diff --git a/.gitignore b/.gitignore index f81086b..b0b248c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ _build/ # coverage.py default output file .coverage + +# IDE cruft +.idea/ diff --git a/docs/performance.rst b/docs/performance.rst index 2d12b9c..d22b06a 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -36,7 +36,7 @@ Both of these approaches will be much faster than using `load/s()`, since they avoid loading the parts of the document we didn't care about. Both `Object` and `Array` have a `mini` property that returns their entire -content as a minified Python `str`. A message router for example would only +content as a minified Python `bytes`. A message router for example would only parse the document and retrieve a single property, the destination, and forward the payload without ever turning it into a Python object. Here's a (bad) example: diff --git a/simdjson/__init__.pyi b/simdjson/__init__.pyi index d42b480..606e804 100644 --- a/simdjson/__init__.pyi +++ b/simdjson/__init__.pyi @@ -52,7 +52,7 @@ class Object(Mapping[str, SimValue]): ... @property - def mini(self) -> str: + def mini(self) -> bytes: ... @@ -73,7 +73,7 @@ class Array(Sequence[SimValue]): ... @property - def mini(self) -> str: + def mini(self) -> bytes: ... diff --git a/simdjson/csimdjson.cpp b/simdjson/csimdjson.cpp index 9145f61..6336efc 100644 --- a/simdjson/csimdjson.cpp +++ b/simdjson/csimdjson.cpp @@ -1,4 +1,4 @@ -/* Generated by Cython 0.29.30 */ +/* Generated by Cython 0.29.32 */ /* BEGIN: Cython Metadata { @@ -32,8 +32,8 @@ END: Cython Metadata */ #elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000) #error Cython requires Python 2.6+ or Python 3.3+. #else -#define CYTHON_ABI "0_29_30" -#define CYTHON_HEX_VERSION 0x001D1EF0 +#define CYTHON_ABI "0_29_32" +#define CYTHON_HEX_VERSION 0x001D20F0 #define CYTHON_FUTURE_DIVISION 1 #include #ifndef offsetof @@ -72,6 +72,7 @@ END: Cython Metadata */ #define CYTHON_COMPILING_IN_PYPY 1 #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_NOGIL 0 #undef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 0 #undef CYTHON_USE_PYTYPE_LOOKUP @@ -115,6 +116,7 @@ END: Cython Metadata */ #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 1 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_NOGIL 0 #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif @@ -155,10 +157,56 @@ END: Cython Metadata */ #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 #endif +#elif defined(PY_NOGIL) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_NOGIL 1 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #ifndef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 #else #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_NOGIL 0 #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif @@ -955,7 +1003,7 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { "simdjson/csimdjson.pyx", "stringsource", - "venv/lib/python3.10/site-packages/Cython/Includes/cpython/type.pxd", + "backend/venv/lib/python3.9/site-packages/Cython/Includes/cpython/type.pxd", }; /* MemviewSliceStruct.proto */ struct __pyx_memoryview_obj; @@ -973,30 +1021,26 @@ typedef struct { #ifndef CYTHON_ATOMICS #define CYTHON_ATOMICS 1 #endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS #define __pyx_atomic_int_type int -#if CYTHON_ATOMICS && __GNUC__ >= 4 && (__GNUC_MINOR__ > 1 ||\ - (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL >= 2)) &&\ - !defined(__i386__) - #define __pyx_atomic_incr_aligned(value, lock) __sync_fetch_and_add(value, 1) - #define __pyx_atomic_decr_aligned(value, lock) __sync_fetch_and_sub(value, 1) +#if CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_incr_aligned(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_aligned(value) __sync_fetch_and_sub(value, 1) #ifdef __PYX_DEBUG_ATOMICS #warning "Using GNU atomics" #endif -#elif CYTHON_ATOMICS && defined(_MSC_VER) && 0 - #include +#elif CYTHON_ATOMICS && defined(_MSC_VER) && CYTHON_COMPILING_IN_NOGIL + #include #undef __pyx_atomic_int_type - #define __pyx_atomic_int_type LONG - #define __pyx_atomic_incr_aligned(value, lock) InterlockedIncrement(value) - #define __pyx_atomic_decr_aligned(value, lock) InterlockedDecrement(value) + #define __pyx_atomic_int_type long + #pragma intrinsic (_InterlockedExchangeAdd) + #define __pyx_atomic_incr_aligned(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_aligned(value) _InterlockedExchangeAdd(value, -1) #ifdef __PYX_DEBUG_ATOMICS #pragma message ("Using MSVC atomics") #endif -#elif CYTHON_ATOMICS && (defined(__ICC) || defined(__INTEL_COMPILER)) && 0 - #define __pyx_atomic_incr_aligned(value, lock) _InterlockedIncrement(value) - #define __pyx_atomic_decr_aligned(value, lock) _InterlockedDecrement(value) - #ifdef __PYX_DEBUG_ATOMICS - #warning "Using Intel atomics" - #endif #else #undef CYTHON_ATOMICS #define CYTHON_ATOMICS 0 @@ -1007,9 +1051,9 @@ typedef struct { typedef volatile __pyx_atomic_int_type __pyx_atomic_int; #if CYTHON_ATOMICS #define __pyx_add_acquisition_count(memview)\ - __pyx_atomic_incr_aligned(__pyx_get_slice_count_pointer(memview), memview->lock) + __pyx_atomic_incr_aligned(__pyx_get_slice_count_pointer(memview)) #define __pyx_sub_acquisition_count(memview)\ - __pyx_atomic_decr_aligned(__pyx_get_slice_count_pointer(memview), memview->lock) + __pyx_atomic_decr_aligned(__pyx_get_slice_count_pointer(memview)) #else #define __pyx_add_acquisition_count(memview)\ __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) @@ -1231,7 +1275,7 @@ struct __pyx_obj_9csimdjson___pyx_scope_struct_4_get_implementations { }; -/* "View.MemoryView":105 +/* "View.MemoryView":106 * * @cname("__pyx_array") * cdef class array: # <<<<<<<<<<<<<< @@ -1256,7 +1300,7 @@ struct __pyx_array_obj { }; -/* "View.MemoryView":279 +/* "View.MemoryView":280 * * @cname('__pyx_MemviewEnum') * cdef class Enum(object): # <<<<<<<<<<<<<< @@ -1269,7 +1313,7 @@ struct __pyx_MemviewEnum_obj { }; -/* "View.MemoryView":330 +/* "View.MemoryView":331 * * @cname('__pyx_memoryview') * cdef class memoryview(object): # <<<<<<<<<<<<<< @@ -1292,7 +1336,7 @@ struct __pyx_memoryview_obj { }; -/* "View.MemoryView":965 +/* "View.MemoryView":967 * * @cname('__pyx_memoryviewslice') * cdef class _memoryviewslice(memoryview): # <<<<<<<<<<<<<< @@ -1354,7 +1398,7 @@ static struct __pyx_vtabstruct_9csimdjson_Object *__pyx_vtabptr_9csimdjson_Objec static CYTHON_INLINE PyObject *__pyx_f_9csimdjson_6Object_from_element(struct __pyx_obj_9csimdjson_Parser *, simdjson::dom::element); -/* "View.MemoryView":105 +/* "View.MemoryView":106 * * @cname("__pyx_array") * cdef class array: # <<<<<<<<<<<<<< @@ -1368,7 +1412,7 @@ struct __pyx_vtabstruct_array { static struct __pyx_vtabstruct_array *__pyx_vtabptr_array; -/* "View.MemoryView":330 +/* "View.MemoryView":331 * * @cname('__pyx_memoryview') * cdef class memoryview(object): # <<<<<<<<<<<<<< @@ -1388,7 +1432,7 @@ struct __pyx_vtabstruct_memoryview { static struct __pyx_vtabstruct_memoryview *__pyx_vtabptr_memoryview; -/* "View.MemoryView":965 +/* "View.MemoryView":967 * * @cname('__pyx_memoryviewslice') * cdef class _memoryviewslice(memoryview): # <<<<<<<<<<<<<< @@ -8676,7 +8720,7 @@ static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_st return __pyx_r; } -/* "View.MemoryView":122 +/* "View.MemoryView":123 * cdef bint dtype_is_object * * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< @@ -8728,13 +8772,13 @@ static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, P case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_itemsize)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, 1); __PYX_ERR(1, 122, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, 1); __PYX_ERR(1, 123, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_format)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, 2); __PYX_ERR(1, 122, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, 2); __PYX_ERR(1, 123, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 3: @@ -8750,7 +8794,7 @@ static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 122, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 123, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -8766,14 +8810,14 @@ static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, P } } __pyx_v_shape = ((PyObject*)values[0]); - __pyx_v_itemsize = __Pyx_PyIndex_AsSsize_t(values[1]); if (unlikely((__pyx_v_itemsize == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 122, __pyx_L3_error) + __pyx_v_itemsize = __Pyx_PyIndex_AsSsize_t(values[1]); if (unlikely((__pyx_v_itemsize == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 123, __pyx_L3_error) __pyx_v_format = values[2]; __pyx_v_mode = values[3]; if (values[4]) { - __pyx_v_allocate_buffer = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_allocate_buffer == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 123, __pyx_L3_error) + __pyx_v_allocate_buffer = __Pyx_PyObject_IsTrue(values[4]); if (unlikely((__pyx_v_allocate_buffer == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 124, __pyx_L3_error) } else { - /* "View.MemoryView":123 + /* "View.MemoryView":124 * * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, * mode="c", bint allocate_buffer=True): # <<<<<<<<<<<<<< @@ -8785,19 +8829,19 @@ static int __pyx_array___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 122, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 3, 5, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 123, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("View.MemoryView.array.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return -1; __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_shape), (&PyTuple_Type), 1, "shape", 1))) __PYX_ERR(1, 122, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_shape), (&PyTuple_Type), 1, "shape", 1))) __PYX_ERR(1, 123, __pyx_L1_error) if (unlikely(((PyObject *)__pyx_v_format) == Py_None)) { - PyErr_Format(PyExc_TypeError, "Argument '%.200s' must not be None", "format"); __PYX_ERR(1, 122, __pyx_L1_error) + PyErr_Format(PyExc_TypeError, "Argument '%.200s' must not be None", "format"); __PYX_ERR(1, 123, __pyx_L1_error) } __pyx_r = __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(((struct __pyx_array_obj *)__pyx_v_self), __pyx_v_shape, __pyx_v_itemsize, __pyx_v_format, __pyx_v_mode, __pyx_v_allocate_buffer); - /* "View.MemoryView":122 + /* "View.MemoryView":123 * cdef bint dtype_is_object * * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< @@ -8839,7 +8883,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __Pyx_RefNannySetupContext("__cinit__", 0); __Pyx_INCREF(__pyx_v_format); - /* "View.MemoryView":129 + /* "View.MemoryView":130 * cdef PyObject **p * * self.ndim = len(shape) # <<<<<<<<<<<<<< @@ -8848,12 +8892,12 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ if (unlikely(__pyx_v_shape == Py_None)) { PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); - __PYX_ERR(1, 129, __pyx_L1_error) + __PYX_ERR(1, 130, __pyx_L1_error) } - __pyx_t_1 = PyTuple_GET_SIZE(__pyx_v_shape); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 129, __pyx_L1_error) + __pyx_t_1 = PyTuple_GET_SIZE(__pyx_v_shape); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 130, __pyx_L1_error) __pyx_v_self->ndim = ((int)__pyx_t_1); - /* "View.MemoryView":130 + /* "View.MemoryView":131 * * self.ndim = len(shape) * self.itemsize = itemsize # <<<<<<<<<<<<<< @@ -8862,7 +8906,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ __pyx_v_self->itemsize = __pyx_v_itemsize; - /* "View.MemoryView":132 + /* "View.MemoryView":133 * self.itemsize = itemsize * * if not self.ndim: # <<<<<<<<<<<<<< @@ -8872,20 +8916,20 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __pyx_t_2 = ((!(__pyx_v_self->ndim != 0)) != 0); if (unlikely(__pyx_t_2)) { - /* "View.MemoryView":133 + /* "View.MemoryView":134 * * if not self.ndim: * raise ValueError("Empty shape tuple for cython.array") # <<<<<<<<<<<<<< * * if itemsize <= 0: */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__16, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 133, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__16, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 134, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 133, __pyx_L1_error) + __PYX_ERR(1, 134, __pyx_L1_error) - /* "View.MemoryView":132 + /* "View.MemoryView":133 * self.itemsize = itemsize * * if not self.ndim: # <<<<<<<<<<<<<< @@ -8894,7 +8938,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ } - /* "View.MemoryView":135 + /* "View.MemoryView":136 * raise ValueError("Empty shape tuple for cython.array") * * if itemsize <= 0: # <<<<<<<<<<<<<< @@ -8904,20 +8948,20 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __pyx_t_2 = ((__pyx_v_itemsize <= 0) != 0); if (unlikely(__pyx_t_2)) { - /* "View.MemoryView":136 + /* "View.MemoryView":137 * * if itemsize <= 0: * raise ValueError("itemsize <= 0 for cython.array") # <<<<<<<<<<<<<< * * if not isinstance(format, bytes): */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__17, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 136, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__17, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 137, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 136, __pyx_L1_error) + __PYX_ERR(1, 137, __pyx_L1_error) - /* "View.MemoryView":135 + /* "View.MemoryView":136 * raise ValueError("Empty shape tuple for cython.array") * * if itemsize <= 0: # <<<<<<<<<<<<<< @@ -8926,7 +8970,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ } - /* "View.MemoryView":138 + /* "View.MemoryView":139 * raise ValueError("itemsize <= 0 for cython.array") * * if not isinstance(format, bytes): # <<<<<<<<<<<<<< @@ -8937,14 +8981,14 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __pyx_t_4 = ((!(__pyx_t_2 != 0)) != 0); if (__pyx_t_4) { - /* "View.MemoryView":139 + /* "View.MemoryView":140 * * if not isinstance(format, bytes): * format = format.encode('ASCII') # <<<<<<<<<<<<<< * self._format = format # keep a reference to the byte string * self.format = self._format */ - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_format, __pyx_n_s_encode); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 139, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_format, __pyx_n_s_encode); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_5))) { @@ -8958,13 +9002,13 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ } __pyx_t_3 = (__pyx_t_6) ? __Pyx_PyObject_Call2Args(__pyx_t_5, __pyx_t_6, __pyx_n_s_ASCII) : __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_n_s_ASCII); __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 139, __pyx_L1_error) + if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF_SET(__pyx_v_format, __pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":138 + /* "View.MemoryView":139 * raise ValueError("itemsize <= 0 for cython.array") * * if not isinstance(format, bytes): # <<<<<<<<<<<<<< @@ -8973,14 +9017,14 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ } - /* "View.MemoryView":140 + /* "View.MemoryView":141 * if not isinstance(format, bytes): * format = format.encode('ASCII') * self._format = format # keep a reference to the byte string # <<<<<<<<<<<<<< * self.format = self._format * */ - if (!(likely(PyBytes_CheckExact(__pyx_v_format))||((__pyx_v_format) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_v_format)->tp_name), 0))) __PYX_ERR(1, 140, __pyx_L1_error) + if (!(likely(PyBytes_CheckExact(__pyx_v_format))||((__pyx_v_format) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_v_format)->tp_name), 0))) __PYX_ERR(1, 141, __pyx_L1_error) __pyx_t_3 = __pyx_v_format; __Pyx_INCREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_3); @@ -8989,7 +9033,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __pyx_v_self->_format = ((PyObject*)__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":141 + /* "View.MemoryView":142 * format = format.encode('ASCII') * self._format = format # keep a reference to the byte string * self.format = self._format # <<<<<<<<<<<<<< @@ -8998,12 +9042,12 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ if (unlikely(__pyx_v_self->_format == Py_None)) { PyErr_SetString(PyExc_TypeError, "expected bytes, NoneType found"); - __PYX_ERR(1, 141, __pyx_L1_error) + __PYX_ERR(1, 142, __pyx_L1_error) } - __pyx_t_7 = __Pyx_PyBytes_AsWritableString(__pyx_v_self->_format); if (unlikely((!__pyx_t_7) && PyErr_Occurred())) __PYX_ERR(1, 141, __pyx_L1_error) + __pyx_t_7 = __Pyx_PyBytes_AsWritableString(__pyx_v_self->_format); if (unlikely((!__pyx_t_7) && PyErr_Occurred())) __PYX_ERR(1, 142, __pyx_L1_error) __pyx_v_self->format = __pyx_t_7; - /* "View.MemoryView":144 + /* "View.MemoryView":145 * * * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) # <<<<<<<<<<<<<< @@ -9012,7 +9056,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ __pyx_v_self->_shape = ((Py_ssize_t *)PyObject_Malloc((((sizeof(Py_ssize_t)) * __pyx_v_self->ndim) * 2))); - /* "View.MemoryView":145 + /* "View.MemoryView":146 * * self._shape = PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2) * self._strides = self._shape + self.ndim # <<<<<<<<<<<<<< @@ -9021,7 +9065,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ __pyx_v_self->_strides = (__pyx_v_self->_shape + __pyx_v_self->ndim); - /* "View.MemoryView":147 + /* "View.MemoryView":148 * self._strides = self._shape + self.ndim * * if not self._shape: # <<<<<<<<<<<<<< @@ -9031,20 +9075,20 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __pyx_t_4 = ((!(__pyx_v_self->_shape != 0)) != 0); if (unlikely(__pyx_t_4)) { - /* "View.MemoryView":148 + /* "View.MemoryView":149 * * if not self._shape: * raise MemoryError("unable to allocate shape and strides.") # <<<<<<<<<<<<<< * * */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__18, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 148, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__18, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 148, __pyx_L1_error) + __PYX_ERR(1, 149, __pyx_L1_error) - /* "View.MemoryView":147 + /* "View.MemoryView":148 * self._strides = self._shape + self.ndim * * if not self._shape: # <<<<<<<<<<<<<< @@ -9053,7 +9097,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ } - /* "View.MemoryView":151 + /* "View.MemoryView":152 * * * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< @@ -9065,18 +9109,18 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ for (;;) { if (__pyx_t_1 >= PyTuple_GET_SIZE(__pyx_t_3)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_5 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_5); __pyx_t_1++; if (unlikely(0 < 0)) __PYX_ERR(1, 151, __pyx_L1_error) + __pyx_t_5 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_5); __pyx_t_1++; if (unlikely(0 < 0)) __PYX_ERR(1, 152, __pyx_L1_error) #else - __pyx_t_5 = PySequence_ITEM(__pyx_t_3, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 151, __pyx_L1_error) + __pyx_t_5 = PySequence_ITEM(__pyx_t_3, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); #endif - __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_5); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 151, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_5); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 152, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_v_dim = __pyx_t_9; __pyx_v_idx = __pyx_t_8; __pyx_t_8 = (__pyx_t_8 + 1); - /* "View.MemoryView":152 + /* "View.MemoryView":153 * * for idx, dim in enumerate(shape): * if dim <= 0: # <<<<<<<<<<<<<< @@ -9086,18 +9130,18 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __pyx_t_4 = ((__pyx_v_dim <= 0) != 0); if (unlikely(__pyx_t_4)) { - /* "View.MemoryView":153 + /* "View.MemoryView":154 * for idx, dim in enumerate(shape): * if dim <= 0: * raise ValueError("Invalid shape in axis %d: %d." % (idx, dim)) # <<<<<<<<<<<<<< * self._shape[idx] = dim * */ - __pyx_t_5 = __Pyx_PyInt_From_int(__pyx_v_idx); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 153, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_int(__pyx_v_idx); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_6 = PyInt_FromSsize_t(__pyx_v_dim); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 153, __pyx_L1_error) + __pyx_t_6 = PyInt_FromSsize_t(__pyx_v_dim); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_10 = PyTuple_New(2); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 153, __pyx_L1_error) + __pyx_t_10 = PyTuple_New(2); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_t_5); @@ -9105,17 +9149,17 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ PyTuple_SET_ITEM(__pyx_t_10, 1, __pyx_t_6); __pyx_t_5 = 0; __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyString_Format(__pyx_kp_s_Invalid_shape_in_axis_d_d, __pyx_t_10); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 153, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyString_Format(__pyx_kp_s_Invalid_shape_in_axis_d_d, __pyx_t_10); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __pyx_t_10 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_6); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 153, __pyx_L1_error) + __pyx_t_10 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_6); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_Raise(__pyx_t_10, 0, 0, 0); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __PYX_ERR(1, 153, __pyx_L1_error) + __PYX_ERR(1, 154, __pyx_L1_error) - /* "View.MemoryView":152 + /* "View.MemoryView":153 * * for idx, dim in enumerate(shape): * if dim <= 0: # <<<<<<<<<<<<<< @@ -9124,7 +9168,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ } - /* "View.MemoryView":154 + /* "View.MemoryView":155 * if dim <= 0: * raise ValueError("Invalid shape in axis %d: %d." % (idx, dim)) * self._shape[idx] = dim # <<<<<<<<<<<<<< @@ -9133,7 +9177,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ (__pyx_v_self->_shape[__pyx_v_idx]) = __pyx_v_dim; - /* "View.MemoryView":151 + /* "View.MemoryView":152 * * * for idx, dim in enumerate(shape): # <<<<<<<<<<<<<< @@ -9143,17 +9187,17 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ } __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":157 + /* "View.MemoryView":158 * * cdef char order * if mode == 'fortran': # <<<<<<<<<<<<<< * order = b'F' * self.mode = u'fortran' */ - __pyx_t_4 = (__Pyx_PyString_Equals(__pyx_v_mode, __pyx_n_s_fortran, Py_EQ)); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_t_4 = (__Pyx_PyString_Equals(__pyx_v_mode, __pyx_n_s_fortran, Py_EQ)); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(1, 158, __pyx_L1_error) if (__pyx_t_4) { - /* "View.MemoryView":158 + /* "View.MemoryView":159 * cdef char order * if mode == 'fortran': * order = b'F' # <<<<<<<<<<<<<< @@ -9162,7 +9206,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ __pyx_v_order = 'F'; - /* "View.MemoryView":159 + /* "View.MemoryView":160 * if mode == 'fortran': * order = b'F' * self.mode = u'fortran' # <<<<<<<<<<<<<< @@ -9175,7 +9219,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __Pyx_DECREF(__pyx_v_self->mode); __pyx_v_self->mode = __pyx_n_u_fortran; - /* "View.MemoryView":157 + /* "View.MemoryView":158 * * cdef char order * if mode == 'fortran': # <<<<<<<<<<<<<< @@ -9185,17 +9229,17 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ goto __pyx_L10; } - /* "View.MemoryView":160 + /* "View.MemoryView":161 * order = b'F' * self.mode = u'fortran' * elif mode == 'c': # <<<<<<<<<<<<<< * order = b'C' * self.mode = u'c' */ - __pyx_t_4 = (__Pyx_PyString_Equals(__pyx_v_mode, __pyx_n_s_c, Py_EQ)); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(1, 160, __pyx_L1_error) + __pyx_t_4 = (__Pyx_PyString_Equals(__pyx_v_mode, __pyx_n_s_c, Py_EQ)); if (unlikely(__pyx_t_4 < 0)) __PYX_ERR(1, 161, __pyx_L1_error) if (likely(__pyx_t_4)) { - /* "View.MemoryView":161 + /* "View.MemoryView":162 * self.mode = u'fortran' * elif mode == 'c': * order = b'C' # <<<<<<<<<<<<<< @@ -9204,7 +9248,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ __pyx_v_order = 'C'; - /* "View.MemoryView":162 + /* "View.MemoryView":163 * elif mode == 'c': * order = b'C' * self.mode = u'c' # <<<<<<<<<<<<<< @@ -9217,7 +9261,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __Pyx_DECREF(__pyx_v_self->mode); __pyx_v_self->mode = __pyx_n_u_c; - /* "View.MemoryView":160 + /* "View.MemoryView":161 * order = b'F' * self.mode = u'fortran' * elif mode == 'c': # <<<<<<<<<<<<<< @@ -9227,7 +9271,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ goto __pyx_L10; } - /* "View.MemoryView":164 + /* "View.MemoryView":165 * self.mode = u'c' * else: * raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode) # <<<<<<<<<<<<<< @@ -9235,18 +9279,18 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ * self.len = fill_contig_strides_array(self._shape, self._strides, */ /*else*/ { - __pyx_t_3 = __Pyx_PyString_FormatSafe(__pyx_kp_s_Invalid_mode_expected_c_or_fortr, __pyx_v_mode); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyString_FormatSafe(__pyx_kp_s_Invalid_mode_expected_c_or_fortr, __pyx_v_mode); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 165, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_10 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_3); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_10 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_3); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 165, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_Raise(__pyx_t_10, 0, 0, 0); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __PYX_ERR(1, 164, __pyx_L1_error) + __PYX_ERR(1, 165, __pyx_L1_error) } __pyx_L10:; - /* "View.MemoryView":166 + /* "View.MemoryView":167 * raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode) * * self.len = fill_contig_strides_array(self._shape, self._strides, # <<<<<<<<<<<<<< @@ -9255,7 +9299,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ __pyx_v_self->len = __pyx_fill_contig_strides_array(__pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_itemsize, __pyx_v_self->ndim, __pyx_v_order); - /* "View.MemoryView":169 + /* "View.MemoryView":170 * itemsize, self.ndim, order) * * self.free_data = allocate_buffer # <<<<<<<<<<<<<< @@ -9264,19 +9308,19 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ __pyx_v_self->free_data = __pyx_v_allocate_buffer; - /* "View.MemoryView":170 + /* "View.MemoryView":171 * * self.free_data = allocate_buffer * self.dtype_is_object = format == b'O' # <<<<<<<<<<<<<< * if allocate_buffer: * */ - __pyx_t_10 = PyObject_RichCompare(__pyx_v_format, __pyx_n_b_O, Py_EQ); __Pyx_XGOTREF(__pyx_t_10); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 170, __pyx_L1_error) - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_10); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 170, __pyx_L1_error) + __pyx_t_10 = PyObject_RichCompare(__pyx_v_format, __pyx_n_b_O, Py_EQ); __Pyx_XGOTREF(__pyx_t_10); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 171, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_10); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 171, __pyx_L1_error) __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; __pyx_v_self->dtype_is_object = __pyx_t_4; - /* "View.MemoryView":171 + /* "View.MemoryView":172 * self.free_data = allocate_buffer * self.dtype_is_object = format == b'O' * if allocate_buffer: # <<<<<<<<<<<<<< @@ -9286,7 +9330,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __pyx_t_4 = (__pyx_v_allocate_buffer != 0); if (__pyx_t_4) { - /* "View.MemoryView":174 + /* "View.MemoryView":175 * * * self.data = malloc(self.len) # <<<<<<<<<<<<<< @@ -9295,7 +9339,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ __pyx_v_self->data = ((char *)malloc(__pyx_v_self->len)); - /* "View.MemoryView":175 + /* "View.MemoryView":176 * * self.data = malloc(self.len) * if not self.data: # <<<<<<<<<<<<<< @@ -9305,20 +9349,20 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __pyx_t_4 = ((!(__pyx_v_self->data != 0)) != 0); if (unlikely(__pyx_t_4)) { - /* "View.MemoryView":176 + /* "View.MemoryView":177 * self.data = malloc(self.len) * if not self.data: * raise MemoryError("unable to allocate array data.") # <<<<<<<<<<<<<< * * if self.dtype_is_object: */ - __pyx_t_10 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__19, NULL); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 176, __pyx_L1_error) + __pyx_t_10 = __Pyx_PyObject_Call(__pyx_builtin_MemoryError, __pyx_tuple__19, NULL); if (unlikely(!__pyx_t_10)) __PYX_ERR(1, 177, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_10); __Pyx_Raise(__pyx_t_10, 0, 0, 0); __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - __PYX_ERR(1, 176, __pyx_L1_error) + __PYX_ERR(1, 177, __pyx_L1_error) - /* "View.MemoryView":175 + /* "View.MemoryView":176 * * self.data = malloc(self.len) * if not self.data: # <<<<<<<<<<<<<< @@ -9327,7 +9371,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ } - /* "View.MemoryView":178 + /* "View.MemoryView":179 * raise MemoryError("unable to allocate array data.") * * if self.dtype_is_object: # <<<<<<<<<<<<<< @@ -9337,7 +9381,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ __pyx_t_4 = (__pyx_v_self->dtype_is_object != 0); if (__pyx_t_4) { - /* "View.MemoryView":179 + /* "View.MemoryView":180 * * if self.dtype_is_object: * p = self.data # <<<<<<<<<<<<<< @@ -9346,7 +9390,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ __pyx_v_p = ((PyObject **)__pyx_v_self->data); - /* "View.MemoryView":180 + /* "View.MemoryView":181 * if self.dtype_is_object: * p = self.data * for i in range(self.len / itemsize): # <<<<<<<<<<<<<< @@ -9355,18 +9399,18 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ if (unlikely(__pyx_v_itemsize == 0)) { PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); - __PYX_ERR(1, 180, __pyx_L1_error) + __PYX_ERR(1, 181, __pyx_L1_error) } else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_itemsize == (Py_ssize_t)-1) && unlikely(UNARY_NEG_WOULD_OVERFLOW(__pyx_v_self->len))) { PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); - __PYX_ERR(1, 180, __pyx_L1_error) + __PYX_ERR(1, 181, __pyx_L1_error) } __pyx_t_1 = __Pyx_div_Py_ssize_t(__pyx_v_self->len, __pyx_v_itemsize); __pyx_t_9 = __pyx_t_1; for (__pyx_t_11 = 0; __pyx_t_11 < __pyx_t_9; __pyx_t_11+=1) { __pyx_v_i = __pyx_t_11; - /* "View.MemoryView":181 + /* "View.MemoryView":182 * p = self.data * for i in range(self.len / itemsize): * p[i] = Py_None # <<<<<<<<<<<<<< @@ -9375,7 +9419,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ (__pyx_v_p[__pyx_v_i]) = Py_None; - /* "View.MemoryView":182 + /* "View.MemoryView":183 * for i in range(self.len / itemsize): * p[i] = Py_None * Py_INCREF(Py_None) # <<<<<<<<<<<<<< @@ -9385,7 +9429,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ Py_INCREF(Py_None); } - /* "View.MemoryView":178 + /* "View.MemoryView":179 * raise MemoryError("unable to allocate array data.") * * if self.dtype_is_object: # <<<<<<<<<<<<<< @@ -9394,7 +9438,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ } - /* "View.MemoryView":171 + /* "View.MemoryView":172 * self.free_data = allocate_buffer * self.dtype_is_object = format == b'O' * if allocate_buffer: # <<<<<<<<<<<<<< @@ -9403,7 +9447,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ */ } - /* "View.MemoryView":122 + /* "View.MemoryView":123 * cdef bint dtype_is_object * * def __cinit__(array self, tuple shape, Py_ssize_t itemsize, format not None, # <<<<<<<<<<<<<< @@ -9427,7 +9471,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array___cinit__(struct __ return __pyx_r; } -/* "View.MemoryView":185 +/* "View.MemoryView":186 * * @cname('getbuffer') * def __getbuffer__(self, Py_buffer *info, int flags): # <<<<<<<<<<<<<< @@ -9470,7 +9514,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); __Pyx_GIVEREF(__pyx_v_info->obj); - /* "View.MemoryView":186 + /* "View.MemoryView":187 * @cname('getbuffer') * def __getbuffer__(self, Py_buffer *info, int flags): * cdef int bufmode = -1 # <<<<<<<<<<<<<< @@ -9479,18 +9523,18 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru */ __pyx_v_bufmode = -1; - /* "View.MemoryView":187 + /* "View.MemoryView":188 * def __getbuffer__(self, Py_buffer *info, int flags): * cdef int bufmode = -1 * if self.mode == u"c": # <<<<<<<<<<<<<< * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS * elif self.mode == u"fortran": */ - __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_n_u_c, Py_EQ)); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 187, __pyx_L1_error) + __pyx_t_1 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_n_u_c, Py_EQ)); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 188, __pyx_L1_error) __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "View.MemoryView":188 + /* "View.MemoryView":189 * cdef int bufmode = -1 * if self.mode == u"c": * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< @@ -9499,7 +9543,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru */ __pyx_v_bufmode = (PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); - /* "View.MemoryView":187 + /* "View.MemoryView":188 * def __getbuffer__(self, Py_buffer *info, int flags): * cdef int bufmode = -1 * if self.mode == u"c": # <<<<<<<<<<<<<< @@ -9509,18 +9553,18 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru goto __pyx_L3; } - /* "View.MemoryView":189 + /* "View.MemoryView":190 * if self.mode == u"c": * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS * elif self.mode == u"fortran": # <<<<<<<<<<<<<< * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS * if not (flags & bufmode): */ - __pyx_t_2 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_n_u_fortran, Py_EQ)); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(1, 189, __pyx_L1_error) + __pyx_t_2 = (__Pyx_PyUnicode_Equals(__pyx_v_self->mode, __pyx_n_u_fortran, Py_EQ)); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(1, 190, __pyx_L1_error) __pyx_t_1 = (__pyx_t_2 != 0); if (__pyx_t_1) { - /* "View.MemoryView":190 + /* "View.MemoryView":191 * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS * elif self.mode == u"fortran": * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< @@ -9529,7 +9573,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru */ __pyx_v_bufmode = (PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS); - /* "View.MemoryView":189 + /* "View.MemoryView":190 * if self.mode == u"c": * bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS * elif self.mode == u"fortran": # <<<<<<<<<<<<<< @@ -9539,7 +9583,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru } __pyx_L3:; - /* "View.MemoryView":191 + /* "View.MemoryView":192 * elif self.mode == u"fortran": * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS * if not (flags & bufmode): # <<<<<<<<<<<<<< @@ -9549,20 +9593,20 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_t_1 = ((!((__pyx_v_flags & __pyx_v_bufmode) != 0)) != 0); if (unlikely(__pyx_t_1)) { - /* "View.MemoryView":192 + /* "View.MemoryView":193 * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS * if not (flags & bufmode): * raise ValueError("Can only create a buffer that is contiguous in memory.") # <<<<<<<<<<<<<< * info.buf = self.data * info.len = self.len */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__20, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 192, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__20, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 193, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 192, __pyx_L1_error) + __PYX_ERR(1, 193, __pyx_L1_error) - /* "View.MemoryView":191 + /* "View.MemoryView":192 * elif self.mode == u"fortran": * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS * if not (flags & bufmode): # <<<<<<<<<<<<<< @@ -9571,7 +9615,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru */ } - /* "View.MemoryView":193 + /* "View.MemoryView":194 * if not (flags & bufmode): * raise ValueError("Can only create a buffer that is contiguous in memory.") * info.buf = self.data # <<<<<<<<<<<<<< @@ -9581,7 +9625,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_t_4 = __pyx_v_self->data; __pyx_v_info->buf = __pyx_t_4; - /* "View.MemoryView":194 + /* "View.MemoryView":195 * raise ValueError("Can only create a buffer that is contiguous in memory.") * info.buf = self.data * info.len = self.len # <<<<<<<<<<<<<< @@ -9591,7 +9635,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_t_5 = __pyx_v_self->len; __pyx_v_info->len = __pyx_t_5; - /* "View.MemoryView":195 + /* "View.MemoryView":196 * info.buf = self.data * info.len = self.len * info.ndim = self.ndim # <<<<<<<<<<<<<< @@ -9601,7 +9645,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_t_6 = __pyx_v_self->ndim; __pyx_v_info->ndim = __pyx_t_6; - /* "View.MemoryView":196 + /* "View.MemoryView":197 * info.len = self.len * info.ndim = self.ndim * info.shape = self._shape # <<<<<<<<<<<<<< @@ -9611,7 +9655,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_t_7 = __pyx_v_self->_shape; __pyx_v_info->shape = __pyx_t_7; - /* "View.MemoryView":197 + /* "View.MemoryView":198 * info.ndim = self.ndim * info.shape = self._shape * info.strides = self._strides # <<<<<<<<<<<<<< @@ -9621,7 +9665,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_t_7 = __pyx_v_self->_strides; __pyx_v_info->strides = __pyx_t_7; - /* "View.MemoryView":198 + /* "View.MemoryView":199 * info.shape = self._shape * info.strides = self._strides * info.suboffsets = NULL # <<<<<<<<<<<<<< @@ -9630,7 +9674,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru */ __pyx_v_info->suboffsets = NULL; - /* "View.MemoryView":199 + /* "View.MemoryView":200 * info.strides = self._strides * info.suboffsets = NULL * info.itemsize = self.itemsize # <<<<<<<<<<<<<< @@ -9640,7 +9684,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_t_5 = __pyx_v_self->itemsize; __pyx_v_info->itemsize = __pyx_t_5; - /* "View.MemoryView":200 + /* "View.MemoryView":201 * info.suboffsets = NULL * info.itemsize = self.itemsize * info.readonly = 0 # <<<<<<<<<<<<<< @@ -9649,7 +9693,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru */ __pyx_v_info->readonly = 0; - /* "View.MemoryView":202 + /* "View.MemoryView":203 * info.readonly = 0 * * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< @@ -9659,7 +9703,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); if (__pyx_t_1) { - /* "View.MemoryView":203 + /* "View.MemoryView":204 * * if flags & PyBUF_FORMAT: * info.format = self.format # <<<<<<<<<<<<<< @@ -9669,7 +9713,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __pyx_t_4 = __pyx_v_self->format; __pyx_v_info->format = __pyx_t_4; - /* "View.MemoryView":202 + /* "View.MemoryView":203 * info.readonly = 0 * * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< @@ -9679,7 +9723,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru goto __pyx_L5; } - /* "View.MemoryView":205 + /* "View.MemoryView":206 * info.format = self.format * else: * info.format = NULL # <<<<<<<<<<<<<< @@ -9691,7 +9735,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru } __pyx_L5:; - /* "View.MemoryView":207 + /* "View.MemoryView":208 * info.format = NULL * * info.obj = self # <<<<<<<<<<<<<< @@ -9704,7 +9748,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = ((PyObject *)__pyx_v_self); - /* "View.MemoryView":185 + /* "View.MemoryView":186 * * @cname('getbuffer') * def __getbuffer__(self, Py_buffer *info, int flags): # <<<<<<<<<<<<<< @@ -9734,7 +9778,7 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_2__getbuffer__(stru return __pyx_r; } -/* "View.MemoryView":211 +/* "View.MemoryView":212 * __pyx_getbuffer = capsule( &__pyx_array_getbuffer, "getbuffer(obj, view, flags)") * * def __dealloc__(array self): # <<<<<<<<<<<<<< @@ -9758,7 +9802,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc int __pyx_t_1; __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "View.MemoryView":212 + /* "View.MemoryView":213 * * def __dealloc__(array self): * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< @@ -9768,7 +9812,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc __pyx_t_1 = ((__pyx_v_self->callback_free_data != NULL) != 0); if (__pyx_t_1) { - /* "View.MemoryView":213 + /* "View.MemoryView":214 * def __dealloc__(array self): * if self.callback_free_data != NULL: * self.callback_free_data(self.data) # <<<<<<<<<<<<<< @@ -9777,7 +9821,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc */ __pyx_v_self->callback_free_data(__pyx_v_self->data); - /* "View.MemoryView":212 + /* "View.MemoryView":213 * * def __dealloc__(array self): * if self.callback_free_data != NULL: # <<<<<<<<<<<<<< @@ -9787,7 +9831,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc goto __pyx_L3; } - /* "View.MemoryView":214 + /* "View.MemoryView":215 * if self.callback_free_data != NULL: * self.callback_free_data(self.data) * elif self.free_data: # <<<<<<<<<<<<<< @@ -9797,7 +9841,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc __pyx_t_1 = (__pyx_v_self->free_data != 0); if (__pyx_t_1) { - /* "View.MemoryView":215 + /* "View.MemoryView":216 * self.callback_free_data(self.data) * elif self.free_data: * if self.dtype_is_object: # <<<<<<<<<<<<<< @@ -9807,7 +9851,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc __pyx_t_1 = (__pyx_v_self->dtype_is_object != 0); if (__pyx_t_1) { - /* "View.MemoryView":216 + /* "View.MemoryView":217 * elif self.free_data: * if self.dtype_is_object: * refcount_objects_in_slice(self.data, self._shape, # <<<<<<<<<<<<<< @@ -9816,7 +9860,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc */ __pyx_memoryview_refcount_objects_in_slice(__pyx_v_self->data, __pyx_v_self->_shape, __pyx_v_self->_strides, __pyx_v_self->ndim, 0); - /* "View.MemoryView":215 + /* "View.MemoryView":216 * self.callback_free_data(self.data) * elif self.free_data: * if self.dtype_is_object: # <<<<<<<<<<<<<< @@ -9825,7 +9869,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc */ } - /* "View.MemoryView":218 + /* "View.MemoryView":219 * refcount_objects_in_slice(self.data, self._shape, * self._strides, self.ndim, False) * free(self.data) # <<<<<<<<<<<<<< @@ -9834,7 +9878,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc */ free(__pyx_v_self->data); - /* "View.MemoryView":214 + /* "View.MemoryView":215 * if self.callback_free_data != NULL: * self.callback_free_data(self.data) * elif self.free_data: # <<<<<<<<<<<<<< @@ -9844,7 +9888,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc } __pyx_L3:; - /* "View.MemoryView":219 + /* "View.MemoryView":220 * self._strides, self.ndim, False) * free(self.data) * PyObject_Free(self._shape) # <<<<<<<<<<<<<< @@ -9853,7 +9897,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc */ PyObject_Free(__pyx_v_self->_shape); - /* "View.MemoryView":211 + /* "View.MemoryView":212 * __pyx_getbuffer = capsule( &__pyx_array_getbuffer, "getbuffer(obj, view, flags)") * * def __dealloc__(array self): # <<<<<<<<<<<<<< @@ -9865,7 +9909,7 @@ static void __pyx_array___pyx_pf_15View_dot_MemoryView_5array_4__dealloc__(struc __Pyx_RefNannyFinishContext(); } -/* "View.MemoryView":222 +/* "View.MemoryView":223 * * @property * def memview(self): # <<<<<<<<<<<<<< @@ -9895,7 +9939,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct _ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":223 + /* "View.MemoryView":224 * @property * def memview(self): * return self.get_memview() # <<<<<<<<<<<<<< @@ -9903,13 +9947,13 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct _ * @cname('get_memview') */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = ((struct __pyx_vtabstruct_array *)__pyx_v_self->__pyx_vtab)->get_memview(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 223, __pyx_L1_error) + __pyx_t_1 = ((struct __pyx_vtabstruct_array *)__pyx_v_self->__pyx_vtab)->get_memview(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 224, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "View.MemoryView":222 + /* "View.MemoryView":223 * * @property * def memview(self): # <<<<<<<<<<<<<< @@ -9928,7 +9972,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_5array_7memview___get__(struct _ return __pyx_r; } -/* "View.MemoryView":226 +/* "View.MemoryView":227 * * @cname('get_memview') * cdef get_memview(self): # <<<<<<<<<<<<<< @@ -9948,7 +9992,7 @@ static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self) { int __pyx_clineno = 0; __Pyx_RefNannySetupContext("get_memview", 0); - /* "View.MemoryView":227 + /* "View.MemoryView":228 * @cname('get_memview') * cdef get_memview(self): * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE # <<<<<<<<<<<<<< @@ -9957,7 +10001,7 @@ static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self) { */ __pyx_v_flags = ((PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT) | PyBUF_WRITABLE); - /* "View.MemoryView":228 + /* "View.MemoryView":229 * cdef get_memview(self): * flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE * return memoryview(self, flags, self.dtype_is_object) # <<<<<<<<<<<<<< @@ -9965,11 +10009,11 @@ static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self) { * def __len__(self): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 228, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 229, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 228, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 229, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 228, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 229, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __Pyx_GIVEREF(((PyObject *)__pyx_v_self)); @@ -9980,14 +10024,14 @@ static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self) { PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_memoryview_type), __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 228, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_memoryview_type), __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 229, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":226 + /* "View.MemoryView":227 * * @cname('get_memview') * cdef get_memview(self): # <<<<<<<<<<<<<< @@ -10008,7 +10052,7 @@ static PyObject *__pyx_array_get_memview(struct __pyx_array_obj *__pyx_v_self) { return __pyx_r; } -/* "View.MemoryView":230 +/* "View.MemoryView":231 * return memoryview(self, flags, self.dtype_is_object) * * def __len__(self): # <<<<<<<<<<<<<< @@ -10034,7 +10078,7 @@ static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(str __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "View.MemoryView":231 + /* "View.MemoryView":232 * * def __len__(self): * return self._shape[0] # <<<<<<<<<<<<<< @@ -10044,7 +10088,7 @@ static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(str __pyx_r = (__pyx_v_self->_shape[0]); goto __pyx_L0; - /* "View.MemoryView":230 + /* "View.MemoryView":231 * return memoryview(self, flags, self.dtype_is_object) * * def __len__(self): # <<<<<<<<<<<<<< @@ -10058,7 +10102,7 @@ static Py_ssize_t __pyx_array___pyx_pf_15View_dot_MemoryView_5array_6__len__(str return __pyx_r; } -/* "View.MemoryView":233 +/* "View.MemoryView":234 * return self._shape[0] * * def __getattr__(self, attr): # <<<<<<<<<<<<<< @@ -10089,7 +10133,7 @@ static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__( int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getattr__", 0); - /* "View.MemoryView":234 + /* "View.MemoryView":235 * * def __getattr__(self, attr): * return getattr(self.memview, attr) # <<<<<<<<<<<<<< @@ -10097,16 +10141,16 @@ static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__( * def __getitem__(self, item): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 234, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 235, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_GetAttr(__pyx_t_1, __pyx_v_attr); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 234, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetAttr(__pyx_t_1, __pyx_v_attr); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 235, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":233 + /* "View.MemoryView":234 * return self._shape[0] * * def __getattr__(self, attr): # <<<<<<<<<<<<<< @@ -10126,7 +10170,7 @@ static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_8__getattr__( return __pyx_r; } -/* "View.MemoryView":236 +/* "View.MemoryView":237 * return getattr(self.memview, attr) * * def __getitem__(self, item): # <<<<<<<<<<<<<< @@ -10157,7 +10201,7 @@ static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getitem__", 0); - /* "View.MemoryView":237 + /* "View.MemoryView":238 * * def __getitem__(self, item): * return self.memview[item] # <<<<<<<<<<<<<< @@ -10165,16 +10209,16 @@ static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__ * def __setitem__(self, item, value): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 237, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 238, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetItem(__pyx_t_1, __pyx_v_item); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 237, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetItem(__pyx_t_1, __pyx_v_item); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 238, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":236 + /* "View.MemoryView":237 * return getattr(self.memview, attr) * * def __getitem__(self, item): # <<<<<<<<<<<<<< @@ -10194,7 +10238,7 @@ static PyObject *__pyx_array___pyx_pf_15View_dot_MemoryView_5array_10__getitem__ return __pyx_r; } -/* "View.MemoryView":239 +/* "View.MemoryView":240 * return self.memview[item] * * def __setitem__(self, item, value): # <<<<<<<<<<<<<< @@ -10224,19 +10268,19 @@ static int __pyx_array___pyx_pf_15View_dot_MemoryView_5array_12__setitem__(struc int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__setitem__", 0); - /* "View.MemoryView":240 + /* "View.MemoryView":241 * * def __setitem__(self, item, value): * self.memview[item] = value # <<<<<<<<<<<<<< * * */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 240, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_memview); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 241, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (unlikely(PyObject_SetItem(__pyx_t_1, __pyx_v_item, __pyx_v_value) < 0)) __PYX_ERR(1, 240, __pyx_L1_error) + if (unlikely(PyObject_SetItem(__pyx_t_1, __pyx_v_item, __pyx_v_value) < 0)) __PYX_ERR(1, 241, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "View.MemoryView":239 + /* "View.MemoryView":240 * return self.memview[item] * * def __setitem__(self, item, value): # <<<<<<<<<<<<<< @@ -10369,7 +10413,7 @@ static PyObject *__pyx_pf___pyx_array_2__setstate_cython__(CYTHON_UNUSED struct return __pyx_r; } -/* "View.MemoryView":244 +/* "View.MemoryView":245 * * @cname("__pyx_array_new") * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, # <<<<<<<<<<<<<< @@ -10391,7 +10435,7 @@ static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize int __pyx_clineno = 0; __Pyx_RefNannySetupContext("array_cwrapper", 0); - /* "View.MemoryView":248 + /* "View.MemoryView":249 * cdef array result * * if buf == NULL: # <<<<<<<<<<<<<< @@ -10401,20 +10445,20 @@ static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize __pyx_t_1 = ((__pyx_v_buf == NULL) != 0); if (__pyx_t_1) { - /* "View.MemoryView":249 + /* "View.MemoryView":250 * * if buf == NULL: * result = array(shape, itemsize, format, mode.decode('ASCII')) # <<<<<<<<<<<<<< * else: * result = array(shape, itemsize, format, mode.decode('ASCII'), */ - __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 249, __pyx_L1_error) + __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 250, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 249, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 250, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_decode_c_string(__pyx_v_mode, 0, strlen(__pyx_v_mode), NULL, NULL, PyUnicode_DecodeASCII); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 249, __pyx_L1_error) + __pyx_t_4 = __Pyx_decode_c_string(__pyx_v_mode, 0, strlen(__pyx_v_mode), NULL, NULL, PyUnicode_DecodeASCII); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 250, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PyTuple_New(4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 249, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 250, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_INCREF(__pyx_v_shape); __Pyx_GIVEREF(__pyx_v_shape); @@ -10428,13 +10472,13 @@ static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize __pyx_t_2 = 0; __pyx_t_3 = 0; __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_array_type), __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 249, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_array_type), __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 250, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":248 + /* "View.MemoryView":249 * cdef array result * * if buf == NULL: # <<<<<<<<<<<<<< @@ -10444,7 +10488,7 @@ static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize goto __pyx_L3; } - /* "View.MemoryView":251 + /* "View.MemoryView":252 * result = array(shape, itemsize, format, mode.decode('ASCII')) * else: * result = array(shape, itemsize, format, mode.decode('ASCII'), # <<<<<<<<<<<<<< @@ -10452,13 +10496,13 @@ static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize * result.data = buf */ /*else*/ { - __pyx_t_4 = PyInt_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 251, __pyx_L1_error) + __pyx_t_4 = PyInt_FromSsize_t(__pyx_v_itemsize); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 252, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 251, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyBytes_FromString(__pyx_v_format); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 252, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_3 = __Pyx_decode_c_string(__pyx_v_mode, 0, strlen(__pyx_v_mode), NULL, NULL, PyUnicode_DecodeASCII); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 251, __pyx_L1_error) + __pyx_t_3 = __Pyx_decode_c_string(__pyx_v_mode, 0, strlen(__pyx_v_mode), NULL, NULL, PyUnicode_DecodeASCII); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 252, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_2 = PyTuple_New(4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 251, __pyx_L1_error) + __pyx_t_2 = PyTuple_New(4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 252, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_v_shape); __Pyx_GIVEREF(__pyx_v_shape); @@ -10473,32 +10517,32 @@ static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize __pyx_t_5 = 0; __pyx_t_3 = 0; - /* "View.MemoryView":252 + /* "View.MemoryView":253 * else: * result = array(shape, itemsize, format, mode.decode('ASCII'), * allocate_buffer=False) # <<<<<<<<<<<<<< * result.data = buf * */ - __pyx_t_3 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 252, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyDict_NewPresized(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 253, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - if (PyDict_SetItem(__pyx_t_3, __pyx_n_s_allocate_buffer, Py_False) < 0) __PYX_ERR(1, 252, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_3, __pyx_n_s_allocate_buffer, Py_False) < 0) __PYX_ERR(1, 253, __pyx_L1_error) - /* "View.MemoryView":251 + /* "View.MemoryView":252 * result = array(shape, itemsize, format, mode.decode('ASCII')) * else: * result = array(shape, itemsize, format, mode.decode('ASCII'), # <<<<<<<<<<<<<< * allocate_buffer=False) * result.data = buf */ - __pyx_t_5 = __Pyx_PyObject_Call(((PyObject *)__pyx_array_type), __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 251, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyObject_Call(((PyObject *)__pyx_array_type), __pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 252, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_v_result = ((struct __pyx_array_obj *)__pyx_t_5); __pyx_t_5 = 0; - /* "View.MemoryView":253 + /* "View.MemoryView":254 * result = array(shape, itemsize, format, mode.decode('ASCII'), * allocate_buffer=False) * result.data = buf # <<<<<<<<<<<<<< @@ -10509,7 +10553,7 @@ static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize } __pyx_L3:; - /* "View.MemoryView":255 + /* "View.MemoryView":256 * result.data = buf * * return result # <<<<<<<<<<<<<< @@ -10521,7 +10565,7 @@ static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize __pyx_r = __pyx_v_result; goto __pyx_L0; - /* "View.MemoryView":244 + /* "View.MemoryView":245 * * @cname("__pyx_array_new") * cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, # <<<<<<<<<<<<<< @@ -10544,7 +10588,7 @@ static struct __pyx_array_obj *__pyx_array_new(PyObject *__pyx_v_shape, Py_ssize return __pyx_r; } -/* "View.MemoryView":281 +/* "View.MemoryView":282 * cdef class Enum(object): * cdef object name * def __init__(self, name): # <<<<<<<<<<<<<< @@ -10581,7 +10625,7 @@ static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_ar else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 281, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 282, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 1) { goto __pyx_L5_argtuple_error; @@ -10592,7 +10636,7 @@ static int __pyx_MemviewEnum___init__(PyObject *__pyx_v_self, PyObject *__pyx_ar } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 281, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 282, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("View.MemoryView.Enum.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -10610,7 +10654,7 @@ static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struc __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__init__", 0); - /* "View.MemoryView":282 + /* "View.MemoryView":283 * cdef object name * def __init__(self, name): * self.name = name # <<<<<<<<<<<<<< @@ -10623,7 +10667,7 @@ static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struc __Pyx_DECREF(__pyx_v_self->name); __pyx_v_self->name = __pyx_v_name; - /* "View.MemoryView":281 + /* "View.MemoryView":282 * cdef class Enum(object): * cdef object name * def __init__(self, name): # <<<<<<<<<<<<<< @@ -10637,7 +10681,7 @@ static int __pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum___init__(struc return __pyx_r; } -/* "View.MemoryView":283 +/* "View.MemoryView":284 * def __init__(self, name): * self.name = name * def __repr__(self): # <<<<<<<<<<<<<< @@ -10663,7 +10707,7 @@ static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr_ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__repr__", 0); - /* "View.MemoryView":284 + /* "View.MemoryView":285 * self.name = name * def __repr__(self): * return self.name # <<<<<<<<<<<<<< @@ -10675,7 +10719,7 @@ static PyObject *__pyx_MemviewEnum___pyx_pf_15View_dot_MemoryView_4Enum_2__repr_ __pyx_r = __pyx_v_self->name; goto __pyx_L0; - /* "View.MemoryView":283 + /* "View.MemoryView":284 * def __init__(self, name): * self.name = name * def __repr__(self): # <<<<<<<<<<<<<< @@ -10983,7 +11027,7 @@ static PyObject *__pyx_pf___pyx_MemviewEnum_2__setstate_cython__(struct __pyx_Me return __pyx_r; } -/* "View.MemoryView":298 +/* "View.MemoryView":299 * * @cname('__pyx_align_pointer') * cdef void *align_pointer(void *memory, size_t alignment) nogil: # <<<<<<<<<<<<<< @@ -10997,7 +11041,7 @@ static void *__pyx_align_pointer(void *__pyx_v_memory, size_t __pyx_v_alignment) void *__pyx_r; int __pyx_t_1; - /* "View.MemoryView":300 + /* "View.MemoryView":301 * cdef void *align_pointer(void *memory, size_t alignment) nogil: * "Align pointer memory on a given boundary" * cdef Py_intptr_t aligned_p = memory # <<<<<<<<<<<<<< @@ -11006,7 +11050,7 @@ static void *__pyx_align_pointer(void *__pyx_v_memory, size_t __pyx_v_alignment) */ __pyx_v_aligned_p = ((Py_intptr_t)__pyx_v_memory); - /* "View.MemoryView":304 + /* "View.MemoryView":305 * * with cython.cdivision(True): * offset = aligned_p % alignment # <<<<<<<<<<<<<< @@ -11015,7 +11059,7 @@ static void *__pyx_align_pointer(void *__pyx_v_memory, size_t __pyx_v_alignment) */ __pyx_v_offset = (__pyx_v_aligned_p % __pyx_v_alignment); - /* "View.MemoryView":306 + /* "View.MemoryView":307 * offset = aligned_p % alignment * * if offset > 0: # <<<<<<<<<<<<<< @@ -11025,7 +11069,7 @@ static void *__pyx_align_pointer(void *__pyx_v_memory, size_t __pyx_v_alignment) __pyx_t_1 = ((__pyx_v_offset > 0) != 0); if (__pyx_t_1) { - /* "View.MemoryView":307 + /* "View.MemoryView":308 * * if offset > 0: * aligned_p += alignment - offset # <<<<<<<<<<<<<< @@ -11034,7 +11078,7 @@ static void *__pyx_align_pointer(void *__pyx_v_memory, size_t __pyx_v_alignment) */ __pyx_v_aligned_p = (__pyx_v_aligned_p + (__pyx_v_alignment - __pyx_v_offset)); - /* "View.MemoryView":306 + /* "View.MemoryView":307 * offset = aligned_p % alignment * * if offset > 0: # <<<<<<<<<<<<<< @@ -11043,7 +11087,7 @@ static void *__pyx_align_pointer(void *__pyx_v_memory, size_t __pyx_v_alignment) */ } - /* "View.MemoryView":309 + /* "View.MemoryView":310 * aligned_p += alignment - offset * * return aligned_p # <<<<<<<<<<<<<< @@ -11053,7 +11097,7 @@ static void *__pyx_align_pointer(void *__pyx_v_memory, size_t __pyx_v_alignment) __pyx_r = ((void *)__pyx_v_aligned_p); goto __pyx_L0; - /* "View.MemoryView":298 + /* "View.MemoryView":299 * * @cname('__pyx_align_pointer') * cdef void *align_pointer(void *memory, size_t alignment) nogil: # <<<<<<<<<<<<<< @@ -11066,7 +11110,7 @@ static void *__pyx_align_pointer(void *__pyx_v_memory, size_t __pyx_v_alignment) return __pyx_r; } -/* "View.MemoryView":345 +/* "View.MemoryView":346 * cdef __Pyx_TypeInfo *typeinfo * * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< @@ -11111,7 +11155,7 @@ static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_ar case 1: if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_flags)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, 1); __PYX_ERR(1, 345, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, 1); __PYX_ERR(1, 346, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: @@ -11121,7 +11165,7 @@ static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_ar } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 345, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) __PYX_ERR(1, 346, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -11134,16 +11178,16 @@ static int __pyx_memoryview___cinit__(PyObject *__pyx_v_self, PyObject *__pyx_ar } } __pyx_v_obj = values[0]; - __pyx_v_flags = __Pyx_PyInt_As_int(values[1]); if (unlikely((__pyx_v_flags == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 345, __pyx_L3_error) + __pyx_v_flags = __Pyx_PyInt_As_int(values[1]); if (unlikely((__pyx_v_flags == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 346, __pyx_L3_error) if (values[2]) { - __pyx_v_dtype_is_object = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_dtype_is_object == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 345, __pyx_L3_error) + __pyx_v_dtype_is_object = __Pyx_PyObject_IsTrue(values[2]); if (unlikely((__pyx_v_dtype_is_object == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 346, __pyx_L3_error) } else { __pyx_v_dtype_is_object = ((int)0); } } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 345, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 346, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("View.MemoryView.memoryview.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -11168,7 +11212,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "View.MemoryView":346 + /* "View.MemoryView":347 * * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): * self.obj = obj # <<<<<<<<<<<<<< @@ -11181,7 +11225,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ __Pyx_DECREF(__pyx_v_self->obj); __pyx_v_self->obj = __pyx_v_obj; - /* "View.MemoryView":347 + /* "View.MemoryView":348 * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): * self.obj = obj * self.flags = flags # <<<<<<<<<<<<<< @@ -11190,7 +11234,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ */ __pyx_v_self->flags = __pyx_v_flags; - /* "View.MemoryView":348 + /* "View.MemoryView":349 * self.obj = obj * self.flags = flags * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< @@ -11210,16 +11254,16 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ __pyx_L4_bool_binop_done:; if (__pyx_t_1) { - /* "View.MemoryView":349 + /* "View.MemoryView":350 * self.flags = flags * if type(self) is memoryview or obj is not None: * __Pyx_GetBuffer(obj, &self.view, flags) # <<<<<<<<<<<<<< * if self.view.obj == NULL: * (<__pyx_buffer *> &self.view).obj = Py_None */ - __pyx_t_4 = __Pyx_GetBuffer(__pyx_v_obj, (&__pyx_v_self->view), __pyx_v_flags); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 349, __pyx_L1_error) + __pyx_t_4 = __Pyx_GetBuffer(__pyx_v_obj, (&__pyx_v_self->view), __pyx_v_flags); if (unlikely(__pyx_t_4 == ((int)-1))) __PYX_ERR(1, 350, __pyx_L1_error) - /* "View.MemoryView":350 + /* "View.MemoryView":351 * if type(self) is memoryview or obj is not None: * __Pyx_GetBuffer(obj, &self.view, flags) * if self.view.obj == NULL: # <<<<<<<<<<<<<< @@ -11229,7 +11273,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ __pyx_t_1 = ((((PyObject *)__pyx_v_self->view.obj) == NULL) != 0); if (__pyx_t_1) { - /* "View.MemoryView":351 + /* "View.MemoryView":352 * __Pyx_GetBuffer(obj, &self.view, flags) * if self.view.obj == NULL: * (<__pyx_buffer *> &self.view).obj = Py_None # <<<<<<<<<<<<<< @@ -11238,16 +11282,16 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ */ ((Py_buffer *)(&__pyx_v_self->view))->obj = Py_None; - /* "View.MemoryView":352 + /* "View.MemoryView":353 * if self.view.obj == NULL: * (<__pyx_buffer *> &self.view).obj = Py_None * Py_INCREF(Py_None) # <<<<<<<<<<<<<< * - * global __pyx_memoryview_thread_locks_used + * if not __PYX_CYTHON_ATOMICS_ENABLED(): */ Py_INCREF(Py_None); - /* "View.MemoryView":350 + /* "View.MemoryView":351 * if type(self) is memoryview or obj is not None: * __Pyx_GetBuffer(obj, &self.view, flags) * if self.view.obj == NULL: # <<<<<<<<<<<<<< @@ -11256,7 +11300,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ */ } - /* "View.MemoryView":348 + /* "View.MemoryView":349 * self.obj = obj * self.flags = flags * if type(self) is memoryview or obj is not None: # <<<<<<<<<<<<<< @@ -11266,100 +11310,119 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ } /* "View.MemoryView":355 + * Py_INCREF(Py_None) * - * global __pyx_memoryview_thread_locks_used - * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: # <<<<<<<<<<<<<< - * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] - * __pyx_memoryview_thread_locks_used += 1 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: */ - __pyx_t_1 = ((__pyx_memoryview_thread_locks_used < 8) != 0); + __pyx_t_1 = ((!(__PYX_CYTHON_ATOMICS_ENABLED() != 0)) != 0); if (__pyx_t_1) { - /* "View.MemoryView":356 - * global __pyx_memoryview_thread_locks_used - * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: - * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] # <<<<<<<<<<<<<< - * __pyx_memoryview_thread_locks_used += 1 - * if self.lock is NULL: - */ - __pyx_v_self->lock = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); - /* "View.MemoryView":357 - * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: - * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] - * __pyx_memoryview_thread_locks_used += 1 # <<<<<<<<<<<<<< - * if self.lock is NULL: - * self.lock = PyThread_allocate_lock() + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: # <<<<<<<<<<<<<< + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 */ - __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used + 1); + __pyx_t_1 = ((__pyx_memoryview_thread_locks_used < 8) != 0); + if (__pyx_t_1) { - /* "View.MemoryView":355 - * - * global __pyx_memoryview_thread_locks_used - * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: # <<<<<<<<<<<<<< - * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] - * __pyx_memoryview_thread_locks_used += 1 + /* "View.MemoryView":358 + * global __pyx_memoryview_thread_locks_used + * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] # <<<<<<<<<<<<<< + * __pyx_memoryview_thread_locks_used += 1 + * if self.lock is NULL: */ - } + __pyx_v_self->lock = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); - /* "View.MemoryView":358 - * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] - * __pyx_memoryview_thread_locks_used += 1 - * if self.lock is NULL: # <<<<<<<<<<<<<< - * self.lock = PyThread_allocate_lock() + /* "View.MemoryView":359 + * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 # <<<<<<<<<<<<<< * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() */ - __pyx_t_1 = ((__pyx_v_self->lock == NULL) != 0); - if (__pyx_t_1) { + __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used + 1); - /* "View.MemoryView":359 - * __pyx_memoryview_thread_locks_used += 1 - * if self.lock is NULL: - * self.lock = PyThread_allocate_lock() # <<<<<<<<<<<<<< - * if self.lock is NULL: - * raise MemoryError + /* "View.MemoryView":357 + * if not __PYX_CYTHON_ATOMICS_ENABLED(): + * global __pyx_memoryview_thread_locks_used + * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: # <<<<<<<<<<<<<< + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 */ - __pyx_v_self->lock = PyThread_allocate_lock(); + } /* "View.MemoryView":360 - * if self.lock is NULL: - * self.lock = PyThread_allocate_lock() + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 * if self.lock is NULL: # <<<<<<<<<<<<<< - * raise MemoryError - * + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: */ __pyx_t_1 = ((__pyx_v_self->lock == NULL) != 0); - if (unlikely(__pyx_t_1)) { + if (__pyx_t_1) { /* "View.MemoryView":361 - * self.lock = PyThread_allocate_lock() + * __pyx_memoryview_thread_locks_used += 1 * if self.lock is NULL: - * raise MemoryError # <<<<<<<<<<<<<< + * self.lock = PyThread_allocate_lock() # <<<<<<<<<<<<<< + * if self.lock is NULL: + * raise MemoryError + */ + __pyx_v_self->lock = PyThread_allocate_lock(); + + /* "View.MemoryView":362 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * + */ + __pyx_t_1 = ((__pyx_v_self->lock == NULL) != 0); + if (unlikely(__pyx_t_1)) { + + /* "View.MemoryView":363 + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: + * raise MemoryError # <<<<<<<<<<<<<< * * if flags & PyBUF_FORMAT: */ - PyErr_NoMemory(); __PYX_ERR(1, 361, __pyx_L1_error) + PyErr_NoMemory(); __PYX_ERR(1, 363, __pyx_L1_error) + + /* "View.MemoryView":362 + * if self.lock is NULL: + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: # <<<<<<<<<<<<<< + * raise MemoryError + * + */ + } /* "View.MemoryView":360 - * if self.lock is NULL: - * self.lock = PyThread_allocate_lock() + * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] + * __pyx_memoryview_thread_locks_used += 1 * if self.lock is NULL: # <<<<<<<<<<<<<< - * raise MemoryError - * + * self.lock = PyThread_allocate_lock() + * if self.lock is NULL: */ } - /* "View.MemoryView":358 - * self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] - * __pyx_memoryview_thread_locks_used += 1 - * if self.lock is NULL: # <<<<<<<<<<<<<< - * self.lock = PyThread_allocate_lock() - * if self.lock is NULL: + /* "View.MemoryView":355 + * Py_INCREF(Py_None) + * + * if not __PYX_CYTHON_ATOMICS_ENABLED(): # <<<<<<<<<<<<<< + * global __pyx_memoryview_thread_locks_used + * if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: */ } - /* "View.MemoryView":363 - * raise MemoryError + /* "View.MemoryView":365 + * raise MemoryError * * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') @@ -11368,7 +11431,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); if (__pyx_t_1) { - /* "View.MemoryView":364 + /* "View.MemoryView":366 * * if flags & PyBUF_FORMAT: * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') # <<<<<<<<<<<<<< @@ -11379,24 +11442,24 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ if (__pyx_t_2) { } else { __pyx_t_1 = __pyx_t_2; - goto __pyx_L11_bool_binop_done; + goto __pyx_L12_bool_binop_done; } __pyx_t_2 = (((__pyx_v_self->view.format[1]) == '\x00') != 0); __pyx_t_1 = __pyx_t_2; - __pyx_L11_bool_binop_done:; + __pyx_L12_bool_binop_done:; __pyx_v_self->dtype_is_object = __pyx_t_1; - /* "View.MemoryView":363 - * raise MemoryError + /* "View.MemoryView":365 + * raise MemoryError * * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') * else: */ - goto __pyx_L10; + goto __pyx_L11; } - /* "View.MemoryView":366 + /* "View.MemoryView":368 * self.dtype_is_object = (self.view.format[0] == b'O' and self.view.format[1] == b'\0') * else: * self.dtype_is_object = dtype_is_object # <<<<<<<<<<<<<< @@ -11406,9 +11469,9 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ /*else*/ { __pyx_v_self->dtype_is_object = __pyx_v_dtype_is_object; } - __pyx_L10:; + __pyx_L11:; - /* "View.MemoryView":368 + /* "View.MemoryView":370 * self.dtype_is_object = dtype_is_object * * self.acquisition_count_aligned_p = <__pyx_atomic_int *> align_pointer( # <<<<<<<<<<<<<< @@ -11417,7 +11480,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ */ __pyx_v_self->acquisition_count_aligned_p = ((__pyx_atomic_int *)__pyx_align_pointer(((void *)(&(__pyx_v_self->acquisition_count[0]))), (sizeof(__pyx_atomic_int)))); - /* "View.MemoryView":370 + /* "View.MemoryView":372 * self.acquisition_count_aligned_p = <__pyx_atomic_int *> align_pointer( * &self.acquisition_count[0], sizeof(__pyx_atomic_int)) * self.typeinfo = NULL # <<<<<<<<<<<<<< @@ -11426,7 +11489,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ */ __pyx_v_self->typeinfo = NULL; - /* "View.MemoryView":345 + /* "View.MemoryView":346 * cdef __Pyx_TypeInfo *typeinfo * * def __cinit__(memoryview self, object obj, int flags, bint dtype_is_object=False): # <<<<<<<<<<<<<< @@ -11445,7 +11508,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview___cinit_ return __pyx_r; } -/* "View.MemoryView":372 +/* "View.MemoryView":374 * self.typeinfo = NULL * * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< @@ -11476,7 +11539,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal PyThread_type_lock __pyx_t_7; __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "View.MemoryView":373 + /* "View.MemoryView":375 * * def __dealloc__(memoryview self): * if self.obj is not None: # <<<<<<<<<<<<<< @@ -11487,7 +11550,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "View.MemoryView":374 + /* "View.MemoryView":376 * def __dealloc__(memoryview self): * if self.obj is not None: * __Pyx_ReleaseBuffer(&self.view) # <<<<<<<<<<<<<< @@ -11496,7 +11559,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal */ __Pyx_ReleaseBuffer((&__pyx_v_self->view)); - /* "View.MemoryView":373 + /* "View.MemoryView":375 * * def __dealloc__(memoryview self): * if self.obj is not None: # <<<<<<<<<<<<<< @@ -11506,7 +11569,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal goto __pyx_L3; } - /* "View.MemoryView":375 + /* "View.MemoryView":377 * if self.obj is not None: * __Pyx_ReleaseBuffer(&self.view) * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< @@ -11516,7 +11579,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal __pyx_t_2 = ((((Py_buffer *)(&__pyx_v_self->view))->obj == Py_None) != 0); if (__pyx_t_2) { - /* "View.MemoryView":377 + /* "View.MemoryView":379 * elif (<__pyx_buffer *> &self.view).obj == Py_None: * * (<__pyx_buffer *> &self.view).obj = NULL # <<<<<<<<<<<<<< @@ -11525,7 +11588,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal */ ((Py_buffer *)(&__pyx_v_self->view))->obj = NULL; - /* "View.MemoryView":378 + /* "View.MemoryView":380 * * (<__pyx_buffer *> &self.view).obj = NULL * Py_DECREF(Py_None) # <<<<<<<<<<<<<< @@ -11534,7 +11597,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal */ Py_DECREF(Py_None); - /* "View.MemoryView":375 + /* "View.MemoryView":377 * if self.obj is not None: * __Pyx_ReleaseBuffer(&self.view) * elif (<__pyx_buffer *> &self.view).obj == Py_None: # <<<<<<<<<<<<<< @@ -11544,7 +11607,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal } __pyx_L3:; - /* "View.MemoryView":382 + /* "View.MemoryView":384 * cdef int i * global __pyx_memoryview_thread_locks_used * if self.lock != NULL: # <<<<<<<<<<<<<< @@ -11554,7 +11617,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal __pyx_t_2 = ((__pyx_v_self->lock != NULL) != 0); if (__pyx_t_2) { - /* "View.MemoryView":383 + /* "View.MemoryView":385 * global __pyx_memoryview_thread_locks_used * if self.lock != NULL: * for i in range(__pyx_memoryview_thread_locks_used): # <<<<<<<<<<<<<< @@ -11566,7 +11629,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { __pyx_v_i = __pyx_t_5; - /* "View.MemoryView":384 + /* "View.MemoryView":386 * if self.lock != NULL: * for i in range(__pyx_memoryview_thread_locks_used): * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< @@ -11576,7 +11639,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal __pyx_t_2 = (((__pyx_memoryview_thread_locks[__pyx_v_i]) == __pyx_v_self->lock) != 0); if (__pyx_t_2) { - /* "View.MemoryView":385 + /* "View.MemoryView":387 * for i in range(__pyx_memoryview_thread_locks_used): * if __pyx_memoryview_thread_locks[i] is self.lock: * __pyx_memoryview_thread_locks_used -= 1 # <<<<<<<<<<<<<< @@ -11585,7 +11648,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal */ __pyx_memoryview_thread_locks_used = (__pyx_memoryview_thread_locks_used - 1); - /* "View.MemoryView":386 + /* "View.MemoryView":388 * if __pyx_memoryview_thread_locks[i] is self.lock: * __pyx_memoryview_thread_locks_used -= 1 * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< @@ -11595,7 +11658,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal __pyx_t_2 = ((__pyx_v_i != __pyx_memoryview_thread_locks_used) != 0); if (__pyx_t_2) { - /* "View.MemoryView":388 + /* "View.MemoryView":390 * if i != __pyx_memoryview_thread_locks_used: * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) # <<<<<<<<<<<<<< @@ -11605,7 +11668,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal __pyx_t_6 = (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]); __pyx_t_7 = (__pyx_memoryview_thread_locks[__pyx_v_i]); - /* "View.MemoryView":387 + /* "View.MemoryView":389 * __pyx_memoryview_thread_locks_used -= 1 * if i != __pyx_memoryview_thread_locks_used: * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( # <<<<<<<<<<<<<< @@ -11615,7 +11678,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal (__pyx_memoryview_thread_locks[__pyx_v_i]) = __pyx_t_6; (__pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used]) = __pyx_t_7; - /* "View.MemoryView":386 + /* "View.MemoryView":388 * if __pyx_memoryview_thread_locks[i] is self.lock: * __pyx_memoryview_thread_locks_used -= 1 * if i != __pyx_memoryview_thread_locks_used: # <<<<<<<<<<<<<< @@ -11624,7 +11687,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal */ } - /* "View.MemoryView":389 + /* "View.MemoryView":391 * __pyx_memoryview_thread_locks[i], __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] = ( * __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used], __pyx_memoryview_thread_locks[i]) * break # <<<<<<<<<<<<<< @@ -11633,7 +11696,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal */ goto __pyx_L6_break; - /* "View.MemoryView":384 + /* "View.MemoryView":386 * if self.lock != NULL: * for i in range(__pyx_memoryview_thread_locks_used): * if __pyx_memoryview_thread_locks[i] is self.lock: # <<<<<<<<<<<<<< @@ -11644,7 +11707,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal } /*else*/ { - /* "View.MemoryView":391 + /* "View.MemoryView":393 * break * else: * PyThread_free_lock(self.lock) # <<<<<<<<<<<<<< @@ -11655,7 +11718,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal } __pyx_L6_break:; - /* "View.MemoryView":382 + /* "View.MemoryView":384 * cdef int i * global __pyx_memoryview_thread_locks_used * if self.lock != NULL: # <<<<<<<<<<<<<< @@ -11664,7 +11727,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal */ } - /* "View.MemoryView":372 + /* "View.MemoryView":374 * self.typeinfo = NULL * * def __dealloc__(memoryview self): # <<<<<<<<<<<<<< @@ -11676,7 +11739,7 @@ static void __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_2__deal __Pyx_RefNannyFinishContext(); } -/* "View.MemoryView":393 +/* "View.MemoryView":395 * PyThread_free_lock(self.lock) * * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< @@ -11702,7 +11765,7 @@ static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__py int __pyx_clineno = 0; __Pyx_RefNannySetupContext("get_item_pointer", 0); - /* "View.MemoryView":395 + /* "View.MemoryView":397 * cdef char *get_item_pointer(memoryview self, object index) except NULL: * cdef Py_ssize_t dim * cdef char *itemp = self.view.buf # <<<<<<<<<<<<<< @@ -11711,7 +11774,7 @@ static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__py */ __pyx_v_itemp = ((char *)__pyx_v_self->view.buf); - /* "View.MemoryView":397 + /* "View.MemoryView":399 * cdef char *itemp = self.view.buf * * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< @@ -11723,26 +11786,26 @@ static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__py __pyx_t_2 = __pyx_v_index; __Pyx_INCREF(__pyx_t_2); __pyx_t_3 = 0; __pyx_t_4 = NULL; } else { - __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_index); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 397, __pyx_L1_error) + __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_index); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 399, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 397, __pyx_L1_error) + __pyx_t_4 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 399, __pyx_L1_error) } for (;;) { if (likely(!__pyx_t_4)) { if (likely(PyList_CheckExact(__pyx_t_2))) { if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_2)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_5 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_5); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(1, 397, __pyx_L1_error) + __pyx_t_5 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_5); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(1, 399, __pyx_L1_error) #else - __pyx_t_5 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 397, __pyx_L1_error) + __pyx_t_5 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 399, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); #endif } else { if (__pyx_t_3 >= PyTuple_GET_SIZE(__pyx_t_2)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_5 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_5); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(1, 397, __pyx_L1_error) + __pyx_t_5 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_5); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(1, 399, __pyx_L1_error) #else - __pyx_t_5 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 397, __pyx_L1_error) + __pyx_t_5 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 399, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); #endif } @@ -11752,7 +11815,7 @@ static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__py PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(1, 397, __pyx_L1_error) + else __PYX_ERR(1, 399, __pyx_L1_error) } break; } @@ -11763,18 +11826,18 @@ static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__py __pyx_v_dim = __pyx_t_1; __pyx_t_1 = (__pyx_t_1 + 1); - /* "View.MemoryView":398 + /* "View.MemoryView":400 * * for dim, idx in enumerate(index): * itemp = pybuffer_index(&self.view, itemp, idx, dim) # <<<<<<<<<<<<<< * * return itemp */ - __pyx_t_6 = __Pyx_PyIndex_AsSsize_t(__pyx_v_idx); if (unlikely((__pyx_t_6 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 398, __pyx_L1_error) - __pyx_t_7 = __pyx_pybuffer_index((&__pyx_v_self->view), __pyx_v_itemp, __pyx_t_6, __pyx_v_dim); if (unlikely(__pyx_t_7 == ((char *)NULL))) __PYX_ERR(1, 398, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyIndex_AsSsize_t(__pyx_v_idx); if (unlikely((__pyx_t_6 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 400, __pyx_L1_error) + __pyx_t_7 = __pyx_pybuffer_index((&__pyx_v_self->view), __pyx_v_itemp, __pyx_t_6, __pyx_v_dim); if (unlikely(__pyx_t_7 == ((char *)NULL))) __PYX_ERR(1, 400, __pyx_L1_error) __pyx_v_itemp = __pyx_t_7; - /* "View.MemoryView":397 + /* "View.MemoryView":399 * cdef char *itemp = self.view.buf * * for dim, idx in enumerate(index): # <<<<<<<<<<<<<< @@ -11784,7 +11847,7 @@ static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__py } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "View.MemoryView":400 + /* "View.MemoryView":402 * itemp = pybuffer_index(&self.view, itemp, idx, dim) * * return itemp # <<<<<<<<<<<<<< @@ -11794,7 +11857,7 @@ static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__py __pyx_r = __pyx_v_itemp; goto __pyx_L0; - /* "View.MemoryView":393 + /* "View.MemoryView":395 * PyThread_free_lock(self.lock) * * cdef char *get_item_pointer(memoryview self, object index) except NULL: # <<<<<<<<<<<<<< @@ -11814,7 +11877,7 @@ static char *__pyx_memoryview_get_item_pointer(struct __pyx_memoryview_obj *__py return __pyx_r; } -/* "View.MemoryView":403 +/* "View.MemoryView":405 * * * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< @@ -11852,7 +11915,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getitem__", 0); - /* "View.MemoryView":404 + /* "View.MemoryView":406 * * def __getitem__(memoryview self, object index): * if index is Ellipsis: # <<<<<<<<<<<<<< @@ -11863,7 +11926,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "View.MemoryView":405 + /* "View.MemoryView":407 * def __getitem__(memoryview self, object index): * if index is Ellipsis: * return self # <<<<<<<<<<<<<< @@ -11875,7 +11938,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "View.MemoryView":404 + /* "View.MemoryView":406 * * def __getitem__(memoryview self, object index): * if index is Ellipsis: # <<<<<<<<<<<<<< @@ -11884,14 +11947,14 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ */ } - /* "View.MemoryView":407 + /* "View.MemoryView":409 * return self * * have_slices, indices = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< * * cdef char *itemp */ - __pyx_t_3 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 407, __pyx_L1_error) + __pyx_t_3 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 409, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); if (likely(__pyx_t_3 != Py_None)) { PyObject* sequence = __pyx_t_3; @@ -11899,7 +11962,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ if (unlikely(size != 2)) { if (size > 2) __Pyx_RaiseTooManyValuesError(2); else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - __PYX_ERR(1, 407, __pyx_L1_error) + __PYX_ERR(1, 409, __pyx_L1_error) } #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); @@ -11907,31 +11970,31 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ __Pyx_INCREF(__pyx_t_4); __Pyx_INCREF(__pyx_t_5); #else - __pyx_t_4 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 407, __pyx_L1_error) + __pyx_t_4 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 409, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 407, __pyx_L1_error) + __pyx_t_5 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 409, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); #endif __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 407, __pyx_L1_error) + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 409, __pyx_L1_error) } __pyx_v_have_slices = __pyx_t_4; __pyx_t_4 = 0; __pyx_v_indices = __pyx_t_5; __pyx_t_5 = 0; - /* "View.MemoryView":410 + /* "View.MemoryView":412 * * cdef char *itemp * if have_slices: # <<<<<<<<<<<<<< * return memview_slice(self, indices) * else: */ - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(1, 410, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(1, 412, __pyx_L1_error) if (__pyx_t_2) { - /* "View.MemoryView":411 + /* "View.MemoryView":413 * cdef char *itemp * if have_slices: * return memview_slice(self, indices) # <<<<<<<<<<<<<< @@ -11939,13 +12002,13 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ * itemp = self.get_item_pointer(indices) */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = ((PyObject *)__pyx_memview_slice(__pyx_v_self, __pyx_v_indices)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 411, __pyx_L1_error) + __pyx_t_3 = ((PyObject *)__pyx_memview_slice(__pyx_v_self, __pyx_v_indices)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 413, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "View.MemoryView":410 + /* "View.MemoryView":412 * * cdef char *itemp * if have_slices: # <<<<<<<<<<<<<< @@ -11954,7 +12017,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ */ } - /* "View.MemoryView":413 + /* "View.MemoryView":415 * return memview_slice(self, indices) * else: * itemp = self.get_item_pointer(indices) # <<<<<<<<<<<<<< @@ -11962,10 +12025,10 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ * */ /*else*/ { - __pyx_t_6 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_indices); if (unlikely(__pyx_t_6 == ((char *)NULL))) __PYX_ERR(1, 413, __pyx_L1_error) + __pyx_t_6 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_indices); if (unlikely(__pyx_t_6 == ((char *)NULL))) __PYX_ERR(1, 415, __pyx_L1_error) __pyx_v_itemp = __pyx_t_6; - /* "View.MemoryView":414 + /* "View.MemoryView":416 * else: * itemp = self.get_item_pointer(indices) * return self.convert_item_to_object(itemp) # <<<<<<<<<<<<<< @@ -11973,14 +12036,14 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ * def __setitem__(memoryview self, object index, object value): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->convert_item_to_object(__pyx_v_self, __pyx_v_itemp); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 414, __pyx_L1_error) + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->convert_item_to_object(__pyx_v_self, __pyx_v_itemp); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 416, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; } - /* "View.MemoryView":403 + /* "View.MemoryView":405 * * * def __getitem__(memoryview self, object index): # <<<<<<<<<<<<<< @@ -12003,7 +12066,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_4_ return __pyx_r; } -/* "View.MemoryView":416 +/* "View.MemoryView":418 * return self.convert_item_to_object(itemp) * * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< @@ -12039,7 +12102,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit __Pyx_RefNannySetupContext("__setitem__", 0); __Pyx_INCREF(__pyx_v_index); - /* "View.MemoryView":417 + /* "View.MemoryView":419 * * def __setitem__(memoryview self, object index, object value): * if self.view.readonly: # <<<<<<<<<<<<<< @@ -12049,20 +12112,20 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit __pyx_t_1 = (__pyx_v_self->view.readonly != 0); if (unlikely(__pyx_t_1)) { - /* "View.MemoryView":418 + /* "View.MemoryView":420 * def __setitem__(memoryview self, object index, object value): * if self.view.readonly: * raise TypeError("Cannot assign to read-only memoryview") # <<<<<<<<<<<<<< * * have_slices, index = _unellipsify(index, self.view.ndim) */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__23, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 418, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__23, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 420, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 418, __pyx_L1_error) + __PYX_ERR(1, 420, __pyx_L1_error) - /* "View.MemoryView":417 + /* "View.MemoryView":419 * * def __setitem__(memoryview self, object index, object value): * if self.view.readonly: # <<<<<<<<<<<<<< @@ -12071,14 +12134,14 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit */ } - /* "View.MemoryView":420 + /* "View.MemoryView":422 * raise TypeError("Cannot assign to read-only memoryview") * * have_slices, index = _unellipsify(index, self.view.ndim) # <<<<<<<<<<<<<< * * if have_slices: */ - __pyx_t_2 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 420, __pyx_L1_error) + __pyx_t_2 = _unellipsify(__pyx_v_index, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 422, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); if (likely(__pyx_t_2 != Py_None)) { PyObject* sequence = __pyx_t_2; @@ -12086,7 +12149,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit if (unlikely(size != 2)) { if (size > 2) __Pyx_RaiseTooManyValuesError(2); else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - __PYX_ERR(1, 420, __pyx_L1_error) + __PYX_ERR(1, 422, __pyx_L1_error) } #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_3 = PyTuple_GET_ITEM(sequence, 0); @@ -12094,67 +12157,67 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(__pyx_t_4); #else - __pyx_t_3 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 420, __pyx_L1_error) + __pyx_t_3 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 422, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 420, __pyx_L1_error) + __pyx_t_4 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 422, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); #endif __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 420, __pyx_L1_error) + __Pyx_RaiseNoneNotIterableError(); __PYX_ERR(1, 422, __pyx_L1_error) } __pyx_v_have_slices = __pyx_t_3; __pyx_t_3 = 0; __Pyx_DECREF_SET(__pyx_v_index, __pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":422 + /* "View.MemoryView":424 * have_slices, index = _unellipsify(index, self.view.ndim) * * if have_slices: # <<<<<<<<<<<<<< * obj = self.is_slice(value) * if obj: */ - __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 422, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_have_slices); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 424, __pyx_L1_error) if (__pyx_t_1) { - /* "View.MemoryView":423 + /* "View.MemoryView":425 * * if have_slices: * obj = self.is_slice(value) # <<<<<<<<<<<<<< * if obj: * self.setitem_slice_assignment(self[index], obj) */ - __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->is_slice(__pyx_v_self, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 423, __pyx_L1_error) + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->is_slice(__pyx_v_self, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 425, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_v_obj = __pyx_t_2; __pyx_t_2 = 0; - /* "View.MemoryView":424 + /* "View.MemoryView":426 * if have_slices: * obj = self.is_slice(value) * if obj: # <<<<<<<<<<<<<< * self.setitem_slice_assignment(self[index], obj) * else: */ - __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_obj); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 424, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_obj); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 426, __pyx_L1_error) if (__pyx_t_1) { - /* "View.MemoryView":425 + /* "View.MemoryView":427 * obj = self.is_slice(value) * if obj: * self.setitem_slice_assignment(self[index], obj) # <<<<<<<<<<<<<< * else: * self.setitem_slice_assign_scalar(self[index], value) */ - __pyx_t_2 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 425, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 427, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_4 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assignment(__pyx_v_self, __pyx_t_2, __pyx_v_obj); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 425, __pyx_L1_error) + __pyx_t_4 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assignment(__pyx_v_self, __pyx_t_2, __pyx_v_obj); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 427, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":424 + /* "View.MemoryView":426 * if have_slices: * obj = self.is_slice(value) * if obj: # <<<<<<<<<<<<<< @@ -12164,7 +12227,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit goto __pyx_L5; } - /* "View.MemoryView":427 + /* "View.MemoryView":429 * self.setitem_slice_assignment(self[index], obj) * else: * self.setitem_slice_assign_scalar(self[index], value) # <<<<<<<<<<<<<< @@ -12172,17 +12235,17 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit * self.setitem_indexed(index, value) */ /*else*/ { - __pyx_t_4 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 427, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetItem(((PyObject *)__pyx_v_self), __pyx_v_index); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 429, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_memoryview_type))))) __PYX_ERR(1, 427, __pyx_L1_error) - __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assign_scalar(__pyx_v_self, ((struct __pyx_memoryview_obj *)__pyx_t_4), __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 427, __pyx_L1_error) + if (!(likely(((__pyx_t_4) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_4, __pyx_memoryview_type))))) __PYX_ERR(1, 429, __pyx_L1_error) + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_slice_assign_scalar(__pyx_v_self, ((struct __pyx_memoryview_obj *)__pyx_t_4), __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 429, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } __pyx_L5:; - /* "View.MemoryView":422 + /* "View.MemoryView":424 * have_slices, index = _unellipsify(index, self.view.ndim) * * if have_slices: # <<<<<<<<<<<<<< @@ -12192,7 +12255,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit goto __pyx_L4; } - /* "View.MemoryView":429 + /* "View.MemoryView":431 * self.setitem_slice_assign_scalar(self[index], value) * else: * self.setitem_indexed(index, value) # <<<<<<<<<<<<<< @@ -12200,13 +12263,13 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit * cdef is_slice(self, obj): */ /*else*/ { - __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_indexed(__pyx_v_self, __pyx_v_index, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 429, __pyx_L1_error) + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->setitem_indexed(__pyx_v_self, __pyx_v_index, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 431, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } __pyx_L4:; - /* "View.MemoryView":416 + /* "View.MemoryView":418 * return self.convert_item_to_object(itemp) * * def __setitem__(memoryview self, object index, object value): # <<<<<<<<<<<<<< @@ -12231,7 +12294,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_6__setit return __pyx_r; } -/* "View.MemoryView":431 +/* "View.MemoryView":433 * self.setitem_indexed(index, value) * * cdef is_slice(self, obj): # <<<<<<<<<<<<<< @@ -12257,7 +12320,7 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ __Pyx_RefNannySetupContext("is_slice", 0); __Pyx_INCREF(__pyx_v_obj); - /* "View.MemoryView":432 + /* "View.MemoryView":434 * * cdef is_slice(self, obj): * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< @@ -12268,7 +12331,7 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ __pyx_t_2 = ((!(__pyx_t_1 != 0)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":433 + /* "View.MemoryView":435 * cdef is_slice(self, obj): * if not isinstance(obj, memoryview): * try: # <<<<<<<<<<<<<< @@ -12284,34 +12347,34 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ __Pyx_XGOTREF(__pyx_t_5); /*try:*/ { - /* "View.MemoryView":434 + /* "View.MemoryView":436 * if not isinstance(obj, memoryview): * try: * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, # <<<<<<<<<<<<<< * self.dtype_is_object) * except TypeError: */ - __pyx_t_6 = __Pyx_PyInt_From_int(((__pyx_v_self->flags & (~PyBUF_WRITABLE)) | PyBUF_ANY_CONTIGUOUS)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 434, __pyx_L4_error) + __pyx_t_6 = __Pyx_PyInt_From_int(((__pyx_v_self->flags & (~PyBUF_WRITABLE)) | PyBUF_ANY_CONTIGUOUS)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 436, __pyx_L4_error) __Pyx_GOTREF(__pyx_t_6); - /* "View.MemoryView":435 + /* "View.MemoryView":437 * try: * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, * self.dtype_is_object) # <<<<<<<<<<<<<< * except TypeError: * return None */ - __pyx_t_7 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 435, __pyx_L4_error) + __pyx_t_7 = __Pyx_PyBool_FromLong(__pyx_v_self->dtype_is_object); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 437, __pyx_L4_error) __Pyx_GOTREF(__pyx_t_7); - /* "View.MemoryView":434 + /* "View.MemoryView":436 * if not isinstance(obj, memoryview): * try: * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, # <<<<<<<<<<<<<< * self.dtype_is_object) * except TypeError: */ - __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 434, __pyx_L4_error) + __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 436, __pyx_L4_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_v_obj); __Pyx_GIVEREF(__pyx_v_obj); @@ -12322,13 +12385,13 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_7); __pyx_t_6 = 0; __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_Call(((PyObject *)__pyx_memoryview_type), __pyx_t_8, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 434, __pyx_L4_error) + __pyx_t_7 = __Pyx_PyObject_Call(((PyObject *)__pyx_memoryview_type), __pyx_t_8, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 436, __pyx_L4_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF_SET(__pyx_v_obj, __pyx_t_7); __pyx_t_7 = 0; - /* "View.MemoryView":433 + /* "View.MemoryView":435 * cdef is_slice(self, obj): * if not isinstance(obj, memoryview): * try: # <<<<<<<<<<<<<< @@ -12345,7 +12408,7 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - /* "View.MemoryView":436 + /* "View.MemoryView":438 * obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, * self.dtype_is_object) * except TypeError: # <<<<<<<<<<<<<< @@ -12355,12 +12418,12 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ __pyx_t_9 = __Pyx_PyErr_ExceptionMatches(__pyx_builtin_TypeError); if (__pyx_t_9) { __Pyx_AddTraceback("View.MemoryView.memoryview.is_slice", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_6) < 0) __PYX_ERR(1, 436, __pyx_L6_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_6) < 0) __PYX_ERR(1, 438, __pyx_L6_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_6); - /* "View.MemoryView":437 + /* "View.MemoryView":439 * self.dtype_is_object) * except TypeError: * return None # <<<<<<<<<<<<<< @@ -12377,7 +12440,7 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ goto __pyx_L6_except_error; __pyx_L6_except_error:; - /* "View.MemoryView":433 + /* "View.MemoryView":435 * cdef is_slice(self, obj): * if not isinstance(obj, memoryview): * try: # <<<<<<<<<<<<<< @@ -12398,7 +12461,7 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ __pyx_L9_try_end:; } - /* "View.MemoryView":432 + /* "View.MemoryView":434 * * cdef is_slice(self, obj): * if not isinstance(obj, memoryview): # <<<<<<<<<<<<<< @@ -12407,7 +12470,7 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ */ } - /* "View.MemoryView":439 + /* "View.MemoryView":441 * return None * * return obj # <<<<<<<<<<<<<< @@ -12419,7 +12482,7 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ __pyx_r = __pyx_v_obj; goto __pyx_L0; - /* "View.MemoryView":431 + /* "View.MemoryView":433 * self.setitem_indexed(index, value) * * cdef is_slice(self, obj): # <<<<<<<<<<<<<< @@ -12441,7 +12504,7 @@ static PyObject *__pyx_memoryview_is_slice(struct __pyx_memoryview_obj *__pyx_v_ return __pyx_r; } -/* "View.MemoryView":441 +/* "View.MemoryView":443 * return obj * * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< @@ -12465,52 +12528,52 @@ static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryvi int __pyx_clineno = 0; __Pyx_RefNannySetupContext("setitem_slice_assignment", 0); - /* "View.MemoryView":445 + /* "View.MemoryView":447 * cdef __Pyx_memviewslice src_slice * * memoryview_copy_contents(get_slice_from_memview(src, &src_slice)[0], # <<<<<<<<<<<<<< * get_slice_from_memview(dst, &dst_slice)[0], * src.ndim, dst.ndim, self.dtype_is_object) */ - if (!(likely(((__pyx_v_src) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_src, __pyx_memoryview_type))))) __PYX_ERR(1, 445, __pyx_L1_error) - __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_src), (&__pyx_v_src_slice)); if (unlikely(__pyx_t_1 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 445, __pyx_L1_error) + if (!(likely(((__pyx_v_src) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_src, __pyx_memoryview_type))))) __PYX_ERR(1, 447, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_src), (&__pyx_v_src_slice)); if (unlikely(__pyx_t_1 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 447, __pyx_L1_error) - /* "View.MemoryView":446 + /* "View.MemoryView":448 * * memoryview_copy_contents(get_slice_from_memview(src, &src_slice)[0], * get_slice_from_memview(dst, &dst_slice)[0], # <<<<<<<<<<<<<< * src.ndim, dst.ndim, self.dtype_is_object) * */ - if (!(likely(((__pyx_v_dst) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_dst, __pyx_memoryview_type))))) __PYX_ERR(1, 446, __pyx_L1_error) - __pyx_t_2 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_dst), (&__pyx_v_dst_slice)); if (unlikely(__pyx_t_2 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 446, __pyx_L1_error) + if (!(likely(((__pyx_v_dst) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_dst, __pyx_memoryview_type))))) __PYX_ERR(1, 448, __pyx_L1_error) + __pyx_t_2 = __pyx_memoryview_get_slice_from_memoryview(((struct __pyx_memoryview_obj *)__pyx_v_dst), (&__pyx_v_dst_slice)); if (unlikely(__pyx_t_2 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 448, __pyx_L1_error) - /* "View.MemoryView":447 + /* "View.MemoryView":449 * memoryview_copy_contents(get_slice_from_memview(src, &src_slice)[0], * get_slice_from_memview(dst, &dst_slice)[0], * src.ndim, dst.ndim, self.dtype_is_object) # <<<<<<<<<<<<<< * * cdef setitem_slice_assign_scalar(self, memoryview dst, value): */ - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_src, __pyx_n_s_ndim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 447, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_src, __pyx_n_s_ndim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 449, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyInt_As_int(__pyx_t_3); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 447, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyInt_As_int(__pyx_t_3); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 449, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_dst, __pyx_n_s_ndim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 447, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_dst, __pyx_n_s_ndim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 449, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = __Pyx_PyInt_As_int(__pyx_t_3); if (unlikely((__pyx_t_5 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 447, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_As_int(__pyx_t_3); if (unlikely((__pyx_t_5 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 449, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":445 + /* "View.MemoryView":447 * cdef __Pyx_memviewslice src_slice * * memoryview_copy_contents(get_slice_from_memview(src, &src_slice)[0], # <<<<<<<<<<<<<< * get_slice_from_memview(dst, &dst_slice)[0], * src.ndim, dst.ndim, self.dtype_is_object) */ - __pyx_t_6 = __pyx_memoryview_copy_contents((__pyx_t_1[0]), (__pyx_t_2[0]), __pyx_t_4, __pyx_t_5, __pyx_v_self->dtype_is_object); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 445, __pyx_L1_error) + __pyx_t_6 = __pyx_memoryview_copy_contents((__pyx_t_1[0]), (__pyx_t_2[0]), __pyx_t_4, __pyx_t_5, __pyx_v_self->dtype_is_object); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 447, __pyx_L1_error) - /* "View.MemoryView":441 + /* "View.MemoryView":443 * return obj * * cdef setitem_slice_assignment(self, dst, src): # <<<<<<<<<<<<<< @@ -12531,7 +12594,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assignment(struct __pyx_memoryvi return __pyx_r; } -/* "View.MemoryView":449 +/* "View.MemoryView":451 * src.ndim, dst.ndim, self.dtype_is_object) * * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< @@ -12564,7 +12627,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor int __pyx_clineno = 0; __Pyx_RefNannySetupContext("setitem_slice_assign_scalar", 0); - /* "View.MemoryView":451 + /* "View.MemoryView":453 * cdef setitem_slice_assign_scalar(self, memoryview dst, value): * cdef int array[128] * cdef void *tmp = NULL # <<<<<<<<<<<<<< @@ -12573,17 +12636,17 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor */ __pyx_v_tmp = NULL; - /* "View.MemoryView":456 + /* "View.MemoryView":458 * cdef __Pyx_memviewslice *dst_slice * cdef __Pyx_memviewslice tmp_slice * dst_slice = get_slice_from_memview(dst, &tmp_slice) # <<<<<<<<<<<<<< * * if self.view.itemsize > sizeof(array): */ - __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_dst, (&__pyx_v_tmp_slice)); if (unlikely(__pyx_t_1 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 456, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_dst, (&__pyx_v_tmp_slice)); if (unlikely(__pyx_t_1 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 458, __pyx_L1_error) __pyx_v_dst_slice = __pyx_t_1; - /* "View.MemoryView":458 + /* "View.MemoryView":460 * dst_slice = get_slice_from_memview(dst, &tmp_slice) * * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< @@ -12593,7 +12656,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor __pyx_t_2 = ((((size_t)__pyx_v_self->view.itemsize) > (sizeof(__pyx_v_array))) != 0); if (__pyx_t_2) { - /* "View.MemoryView":459 + /* "View.MemoryView":461 * * if self.view.itemsize > sizeof(array): * tmp = PyMem_Malloc(self.view.itemsize) # <<<<<<<<<<<<<< @@ -12602,7 +12665,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor */ __pyx_v_tmp = PyMem_Malloc(__pyx_v_self->view.itemsize); - /* "View.MemoryView":460 + /* "View.MemoryView":462 * if self.view.itemsize > sizeof(array): * tmp = PyMem_Malloc(self.view.itemsize) * if tmp == NULL: # <<<<<<<<<<<<<< @@ -12612,16 +12675,16 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor __pyx_t_2 = ((__pyx_v_tmp == NULL) != 0); if (unlikely(__pyx_t_2)) { - /* "View.MemoryView":461 + /* "View.MemoryView":463 * tmp = PyMem_Malloc(self.view.itemsize) * if tmp == NULL: * raise MemoryError # <<<<<<<<<<<<<< * item = tmp * else: */ - PyErr_NoMemory(); __PYX_ERR(1, 461, __pyx_L1_error) + PyErr_NoMemory(); __PYX_ERR(1, 463, __pyx_L1_error) - /* "View.MemoryView":460 + /* "View.MemoryView":462 * if self.view.itemsize > sizeof(array): * tmp = PyMem_Malloc(self.view.itemsize) * if tmp == NULL: # <<<<<<<<<<<<<< @@ -12630,7 +12693,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor */ } - /* "View.MemoryView":462 + /* "View.MemoryView":464 * if tmp == NULL: * raise MemoryError * item = tmp # <<<<<<<<<<<<<< @@ -12639,7 +12702,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor */ __pyx_v_item = __pyx_v_tmp; - /* "View.MemoryView":458 + /* "View.MemoryView":460 * dst_slice = get_slice_from_memview(dst, &tmp_slice) * * if self.view.itemsize > sizeof(array): # <<<<<<<<<<<<<< @@ -12649,7 +12712,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor goto __pyx_L3; } - /* "View.MemoryView":464 + /* "View.MemoryView":466 * item = tmp * else: * item = array # <<<<<<<<<<<<<< @@ -12661,7 +12724,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor } __pyx_L3:; - /* "View.MemoryView":466 + /* "View.MemoryView":468 * item = array * * try: # <<<<<<<<<<<<<< @@ -12670,7 +12733,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor */ /*try:*/ { - /* "View.MemoryView":467 + /* "View.MemoryView":469 * * try: * if self.dtype_is_object: # <<<<<<<<<<<<<< @@ -12680,7 +12743,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor __pyx_t_2 = (__pyx_v_self->dtype_is_object != 0); if (__pyx_t_2) { - /* "View.MemoryView":468 + /* "View.MemoryView":470 * try: * if self.dtype_is_object: * ( item)[0] = value # <<<<<<<<<<<<<< @@ -12689,7 +12752,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor */ (((PyObject **)__pyx_v_item)[0]) = ((PyObject *)__pyx_v_value); - /* "View.MemoryView":467 + /* "View.MemoryView":469 * * try: * if self.dtype_is_object: # <<<<<<<<<<<<<< @@ -12699,7 +12762,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor goto __pyx_L8; } - /* "View.MemoryView":470 + /* "View.MemoryView":472 * ( item)[0] = value * else: * self.assign_item_from_object( item, value) # <<<<<<<<<<<<<< @@ -12707,13 +12770,13 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor * */ /*else*/ { - __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, ((char *)__pyx_v_item), __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 470, __pyx_L6_error) + __pyx_t_3 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, ((char *)__pyx_v_item), __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 472, __pyx_L6_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } __pyx_L8:; - /* "View.MemoryView":474 + /* "View.MemoryView":476 * * * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< @@ -12723,18 +12786,18 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor __pyx_t_2 = ((__pyx_v_self->view.suboffsets != NULL) != 0); if (__pyx_t_2) { - /* "View.MemoryView":475 + /* "View.MemoryView":477 * * if self.view.suboffsets != NULL: * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) # <<<<<<<<<<<<<< * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, * item, self.dtype_is_object) */ - __pyx_t_3 = assert_direct_dimensions(__pyx_v_self->view.suboffsets, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 475, __pyx_L6_error) + __pyx_t_3 = assert_direct_dimensions(__pyx_v_self->view.suboffsets, __pyx_v_self->view.ndim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 477, __pyx_L6_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":474 + /* "View.MemoryView":476 * * * if self.view.suboffsets != NULL: # <<<<<<<<<<<<<< @@ -12743,7 +12806,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor */ } - /* "View.MemoryView":476 + /* "View.MemoryView":478 * if self.view.suboffsets != NULL: * assert_direct_dimensions(self.view.suboffsets, self.view.ndim) * slice_assign_scalar(dst_slice, dst.view.ndim, self.view.itemsize, # <<<<<<<<<<<<<< @@ -12753,7 +12816,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor __pyx_memoryview_slice_assign_scalar(__pyx_v_dst_slice, __pyx_v_dst->view.ndim, __pyx_v_self->view.itemsize, __pyx_v_item, __pyx_v_self->dtype_is_object); } - /* "View.MemoryView":479 + /* "View.MemoryView":481 * item, self.dtype_is_object) * finally: * PyMem_Free(tmp) # <<<<<<<<<<<<<< @@ -12800,7 +12863,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor __pyx_L7:; } - /* "View.MemoryView":449 + /* "View.MemoryView":451 * src.ndim, dst.ndim, self.dtype_is_object) * * cdef setitem_slice_assign_scalar(self, memoryview dst, value): # <<<<<<<<<<<<<< @@ -12821,7 +12884,7 @@ static PyObject *__pyx_memoryview_setitem_slice_assign_scalar(struct __pyx_memor return __pyx_r; } -/* "View.MemoryView":481 +/* "View.MemoryView":483 * PyMem_Free(tmp) * * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< @@ -12840,28 +12903,28 @@ static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("setitem_indexed", 0); - /* "View.MemoryView":482 + /* "View.MemoryView":484 * * cdef setitem_indexed(self, index, value): * cdef char *itemp = self.get_item_pointer(index) # <<<<<<<<<<<<<< * self.assign_item_from_object(itemp, value) * */ - __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_index); if (unlikely(__pyx_t_1 == ((char *)NULL))) __PYX_ERR(1, 482, __pyx_L1_error) + __pyx_t_1 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->get_item_pointer(__pyx_v_self, __pyx_v_index); if (unlikely(__pyx_t_1 == ((char *)NULL))) __PYX_ERR(1, 484, __pyx_L1_error) __pyx_v_itemp = __pyx_t_1; - /* "View.MemoryView":483 + /* "View.MemoryView":485 * cdef setitem_indexed(self, index, value): * cdef char *itemp = self.get_item_pointer(index) * self.assign_item_from_object(itemp, value) # <<<<<<<<<<<<<< * * cdef convert_item_to_object(self, char *itemp): */ - __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 483, __pyx_L1_error) + __pyx_t_2 = ((struct __pyx_vtabstruct_memoryview *)__pyx_v_self->__pyx_vtab)->assign_item_from_object(__pyx_v_self, __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 485, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "View.MemoryView":481 + /* "View.MemoryView":483 * PyMem_Free(tmp) * * cdef setitem_indexed(self, index, value): # <<<<<<<<<<<<<< @@ -12882,7 +12945,7 @@ static PyObject *__pyx_memoryview_setitem_indexed(struct __pyx_memoryview_obj *_ return __pyx_r; } -/* "View.MemoryView":485 +/* "View.MemoryView":487 * self.assign_item_from_object(itemp, value) * * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< @@ -12912,31 +12975,31 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview int __pyx_clineno = 0; __Pyx_RefNannySetupContext("convert_item_to_object", 0); - /* "View.MemoryView":488 + /* "View.MemoryView":490 * """Only used if instantiated manually by the user, or if Cython doesn't * know how to convert the type""" * import struct # <<<<<<<<<<<<<< * cdef bytes bytesitem * */ - __pyx_t_1 = __Pyx_Import(__pyx_n_s_struct, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 488, __pyx_L1_error) + __pyx_t_1 = __Pyx_Import(__pyx_n_s_struct, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 490, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_struct = __pyx_t_1; __pyx_t_1 = 0; - /* "View.MemoryView":491 + /* "View.MemoryView":493 * cdef bytes bytesitem * * bytesitem = itemp[:self.view.itemsize] # <<<<<<<<<<<<<< * try: * result = struct.unpack(self.view.format, bytesitem) */ - __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_itemp + 0, __pyx_v_self->view.itemsize - 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 491, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_itemp + 0, __pyx_v_self->view.itemsize - 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 493, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_bytesitem = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "View.MemoryView":492 + /* "View.MemoryView":494 * * bytesitem = itemp[:self.view.itemsize] * try: # <<<<<<<<<<<<<< @@ -12952,16 +13015,16 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview __Pyx_XGOTREF(__pyx_t_4); /*try:*/ { - /* "View.MemoryView":493 + /* "View.MemoryView":495 * bytesitem = itemp[:self.view.itemsize] * try: * result = struct.unpack(self.view.format, bytesitem) # <<<<<<<<<<<<<< * except struct.error: * raise ValueError("Unable to convert item to object") */ - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_n_s_unpack); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 493, __pyx_L3_error) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_n_s_unpack); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 495, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_6 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 493, __pyx_L3_error) + __pyx_t_6 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 495, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_7 = NULL; __pyx_t_8 = 0; @@ -12978,7 +13041,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[3] = {__pyx_t_7, __pyx_t_6, __pyx_v_bytesitem}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_8, 2+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 493, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_8, 2+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 495, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -12987,14 +13050,14 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[3] = {__pyx_t_7, __pyx_t_6, __pyx_v_bytesitem}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_8, 2+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 493, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_8, 2+__pyx_t_8); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 495, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { - __pyx_t_9 = PyTuple_New(2+__pyx_t_8); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 493, __pyx_L3_error) + __pyx_t_9 = PyTuple_New(2+__pyx_t_8); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 495, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); if (__pyx_t_7) { __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_9, 0, __pyx_t_7); __pyx_t_7 = NULL; @@ -13005,7 +13068,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview __Pyx_GIVEREF(__pyx_v_bytesitem); PyTuple_SET_ITEM(__pyx_t_9, 1+__pyx_t_8, __pyx_v_bytesitem); __pyx_t_6 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 493, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 495, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } @@ -13013,7 +13076,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview __pyx_v_result = __pyx_t_1; __pyx_t_1 = 0; - /* "View.MemoryView":492 + /* "View.MemoryView":494 * * bytesitem = itemp[:self.view.itemsize] * try: # <<<<<<<<<<<<<< @@ -13022,7 +13085,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview */ } - /* "View.MemoryView":497 + /* "View.MemoryView":499 * raise ValueError("Unable to convert item to object") * else: * if len(self.view.format) == 1: # <<<<<<<<<<<<<< @@ -13034,7 +13097,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview __pyx_t_11 = ((__pyx_t_10 == 1) != 0); if (__pyx_t_11) { - /* "View.MemoryView":498 + /* "View.MemoryView":500 * else: * if len(self.view.format) == 1: * return result[0] # <<<<<<<<<<<<<< @@ -13042,13 +13105,13 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_result, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 498, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_GetItemInt(__pyx_v_result, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 500, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L6_except_return; - /* "View.MemoryView":497 + /* "View.MemoryView":499 * raise ValueError("Unable to convert item to object") * else: * if len(self.view.format) == 1: # <<<<<<<<<<<<<< @@ -13057,7 +13120,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview */ } - /* "View.MemoryView":499 + /* "View.MemoryView":501 * if len(self.view.format) == 1: * return result[0] * return result # <<<<<<<<<<<<<< @@ -13076,7 +13139,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; - /* "View.MemoryView":494 + /* "View.MemoryView":496 * try: * result = struct.unpack(self.view.format, bytesitem) * except struct.error: # <<<<<<<<<<<<<< @@ -13084,7 +13147,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview * else: */ __Pyx_ErrFetch(&__pyx_t_1, &__pyx_t_5, &__pyx_t_9); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_n_s_error); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 494, __pyx_L5_except_error) + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_n_s_error); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 496, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_8 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_1, __pyx_t_6); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -13092,28 +13155,28 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview __pyx_t_1 = 0; __pyx_t_5 = 0; __pyx_t_9 = 0; if (__pyx_t_8) { __Pyx_AddTraceback("View.MemoryView.memoryview.convert_item_to_object", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_9, &__pyx_t_5, &__pyx_t_1) < 0) __PYX_ERR(1, 494, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_9, &__pyx_t_5, &__pyx_t_1) < 0) __PYX_ERR(1, 496, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_9); __Pyx_GOTREF(__pyx_t_5); __Pyx_GOTREF(__pyx_t_1); - /* "View.MemoryView":495 + /* "View.MemoryView":497 * result = struct.unpack(self.view.format, bytesitem) * except struct.error: * raise ValueError("Unable to convert item to object") # <<<<<<<<<<<<<< * else: * if len(self.view.format) == 1: */ - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__24, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 495, __pyx_L5_except_error) + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__24, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 497, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_Raise(__pyx_t_6, 0, 0, 0); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __PYX_ERR(1, 495, __pyx_L5_except_error) + __PYX_ERR(1, 497, __pyx_L5_except_error) } goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "View.MemoryView":492 + /* "View.MemoryView":494 * * bytesitem = itemp[:self.view.itemsize] * try: # <<<<<<<<<<<<<< @@ -13133,7 +13196,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview goto __pyx_L0; } - /* "View.MemoryView":485 + /* "View.MemoryView":487 * self.assign_item_from_object(itemp, value) * * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< @@ -13159,7 +13222,7 @@ static PyObject *__pyx_memoryview_convert_item_to_object(struct __pyx_memoryview return __pyx_r; } -/* "View.MemoryView":501 +/* "View.MemoryView":503 * return result * * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< @@ -13193,19 +13256,19 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie int __pyx_clineno = 0; __Pyx_RefNannySetupContext("assign_item_from_object", 0); - /* "View.MemoryView":504 + /* "View.MemoryView":506 * """Only used if instantiated manually by the user, or if Cython doesn't * know how to convert the type""" * import struct # <<<<<<<<<<<<<< * cdef char c * cdef bytes bytesvalue */ - __pyx_t_1 = __Pyx_Import(__pyx_n_s_struct, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 504, __pyx_L1_error) + __pyx_t_1 = __Pyx_Import(__pyx_n_s_struct, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 506, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_struct = __pyx_t_1; __pyx_t_1 = 0; - /* "View.MemoryView":509 + /* "View.MemoryView":511 * cdef Py_ssize_t i * * if isinstance(value, tuple): # <<<<<<<<<<<<<< @@ -13216,37 +13279,37 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie __pyx_t_3 = (__pyx_t_2 != 0); if (__pyx_t_3) { - /* "View.MemoryView":510 + /* "View.MemoryView":512 * * if isinstance(value, tuple): * bytesvalue = struct.pack(self.view.format, *value) # <<<<<<<<<<<<<< * else: * bytesvalue = struct.pack(self.view.format, value) */ - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_n_s_pack); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 510, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_n_s_pack); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 512, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 510, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 512, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 510, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 512, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PySequence_Tuple(__pyx_v_value); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 510, __pyx_L1_error) + __pyx_t_4 = __Pyx_PySequence_Tuple(__pyx_v_value); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 512, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_6 = PyNumber_Add(__pyx_t_5, __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 510, __pyx_L1_error) + __pyx_t_6 = PyNumber_Add(__pyx_t_5, __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 512, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 510, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 512, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_4)->tp_name), 0))) __PYX_ERR(1, 510, __pyx_L1_error) + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_4)->tp_name), 0))) __PYX_ERR(1, 512, __pyx_L1_error) __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":509 + /* "View.MemoryView":511 * cdef Py_ssize_t i * * if isinstance(value, tuple): # <<<<<<<<<<<<<< @@ -13256,7 +13319,7 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie goto __pyx_L3; } - /* "View.MemoryView":512 + /* "View.MemoryView":514 * bytesvalue = struct.pack(self.view.format, *value) * else: * bytesvalue = struct.pack(self.view.format, value) # <<<<<<<<<<<<<< @@ -13264,9 +13327,9 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie * for i, c in enumerate(bytesvalue): */ /*else*/ { - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_n_s_pack); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 512, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_struct, __pyx_n_s_pack); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 514, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_1 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 512, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBytes_FromString(__pyx_v_self->view.format); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 514, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_5 = NULL; __pyx_t_7 = 0; @@ -13283,7 +13346,7 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[3] = {__pyx_t_5, __pyx_t_1, __pyx_v_value}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-__pyx_t_7, 2+__pyx_t_7); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 512, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-__pyx_t_7, 2+__pyx_t_7); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 514, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -13292,14 +13355,14 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[3] = {__pyx_t_5, __pyx_t_1, __pyx_v_value}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-__pyx_t_7, 2+__pyx_t_7); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 512, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-__pyx_t_7, 2+__pyx_t_7); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 514, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; } else #endif { - __pyx_t_8 = PyTuple_New(2+__pyx_t_7); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 512, __pyx_L1_error) + __pyx_t_8 = PyTuple_New(2+__pyx_t_7); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 514, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_8); if (__pyx_t_5) { __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_5); __pyx_t_5 = NULL; @@ -13310,18 +13373,18 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie __Pyx_GIVEREF(__pyx_v_value); PyTuple_SET_ITEM(__pyx_t_8, 1+__pyx_t_7, __pyx_v_value); __pyx_t_1 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 512, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 514, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_4)->tp_name), 0))) __PYX_ERR(1, 512, __pyx_L1_error) + if (!(likely(PyBytes_CheckExact(__pyx_t_4))||((__pyx_t_4) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_4)->tp_name), 0))) __PYX_ERR(1, 514, __pyx_L1_error) __pyx_v_bytesvalue = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; } __pyx_L3:; - /* "View.MemoryView":514 + /* "View.MemoryView":516 * bytesvalue = struct.pack(self.view.format, value) * * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< @@ -13331,7 +13394,7 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie __pyx_t_9 = 0; if (unlikely(__pyx_v_bytesvalue == Py_None)) { PyErr_SetString(PyExc_TypeError, "'NoneType' is not iterable"); - __PYX_ERR(1, 514, __pyx_L1_error) + __PYX_ERR(1, 516, __pyx_L1_error) } __Pyx_INCREF(__pyx_v_bytesvalue); __pyx_t_10 = __pyx_v_bytesvalue; @@ -13341,7 +13404,7 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie __pyx_t_11 = __pyx_t_14; __pyx_v_c = (__pyx_t_11[0]); - /* "View.MemoryView":515 + /* "View.MemoryView":517 * * for i, c in enumerate(bytesvalue): * itemp[i] = c # <<<<<<<<<<<<<< @@ -13350,7 +13413,7 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie */ __pyx_v_i = __pyx_t_9; - /* "View.MemoryView":514 + /* "View.MemoryView":516 * bytesvalue = struct.pack(self.view.format, value) * * for i, c in enumerate(bytesvalue): # <<<<<<<<<<<<<< @@ -13359,7 +13422,7 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie */ __pyx_t_9 = (__pyx_t_9 + 1); - /* "View.MemoryView":515 + /* "View.MemoryView":517 * * for i, c in enumerate(bytesvalue): * itemp[i] = c # <<<<<<<<<<<<<< @@ -13370,7 +13433,7 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie } __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; - /* "View.MemoryView":501 + /* "View.MemoryView":503 * return result * * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< @@ -13398,7 +13461,7 @@ static PyObject *__pyx_memoryview_assign_item_from_object(struct __pyx_memoryvie return __pyx_r; } -/* "View.MemoryView":518 +/* "View.MemoryView":520 * * @cname('getbuffer') * def __getbuffer__(self, Py_buffer *info, int flags): # <<<<<<<<<<<<<< @@ -13441,7 +13504,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_v_info->obj = Py_None; __Pyx_INCREF(Py_None); __Pyx_GIVEREF(__pyx_v_info->obj); - /* "View.MemoryView":519 + /* "View.MemoryView":521 * @cname('getbuffer') * def __getbuffer__(self, Py_buffer *info, int flags): * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< @@ -13459,20 +13522,20 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_L4_bool_binop_done:; if (unlikely(__pyx_t_1)) { - /* "View.MemoryView":520 + /* "View.MemoryView":522 * def __getbuffer__(self, Py_buffer *info, int flags): * if flags & PyBUF_WRITABLE and self.view.readonly: * raise ValueError("Cannot create writable memory view from read-only memoryview") # <<<<<<<<<<<<<< * * if flags & PyBUF_ND: */ - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__25, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 520, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__25, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 522, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 520, __pyx_L1_error) + __PYX_ERR(1, 522, __pyx_L1_error) - /* "View.MemoryView":519 + /* "View.MemoryView":521 * @cname('getbuffer') * def __getbuffer__(self, Py_buffer *info, int flags): * if flags & PyBUF_WRITABLE and self.view.readonly: # <<<<<<<<<<<<<< @@ -13481,7 +13544,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu */ } - /* "View.MemoryView":522 + /* "View.MemoryView":524 * raise ValueError("Cannot create writable memory view from read-only memoryview") * * if flags & PyBUF_ND: # <<<<<<<<<<<<<< @@ -13491,7 +13554,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_1 = ((__pyx_v_flags & PyBUF_ND) != 0); if (__pyx_t_1) { - /* "View.MemoryView":523 + /* "View.MemoryView":525 * * if flags & PyBUF_ND: * info.shape = self.view.shape # <<<<<<<<<<<<<< @@ -13501,7 +13564,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_4 = __pyx_v_self->view.shape; __pyx_v_info->shape = __pyx_t_4; - /* "View.MemoryView":522 + /* "View.MemoryView":524 * raise ValueError("Cannot create writable memory view from read-only memoryview") * * if flags & PyBUF_ND: # <<<<<<<<<<<<<< @@ -13511,7 +13574,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu goto __pyx_L6; } - /* "View.MemoryView":525 + /* "View.MemoryView":527 * info.shape = self.view.shape * else: * info.shape = NULL # <<<<<<<<<<<<<< @@ -13523,7 +13586,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu } __pyx_L6:; - /* "View.MemoryView":527 + /* "View.MemoryView":529 * info.shape = NULL * * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< @@ -13533,7 +13596,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_1 = ((__pyx_v_flags & PyBUF_STRIDES) != 0); if (__pyx_t_1) { - /* "View.MemoryView":528 + /* "View.MemoryView":530 * * if flags & PyBUF_STRIDES: * info.strides = self.view.strides # <<<<<<<<<<<<<< @@ -13543,7 +13606,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_4 = __pyx_v_self->view.strides; __pyx_v_info->strides = __pyx_t_4; - /* "View.MemoryView":527 + /* "View.MemoryView":529 * info.shape = NULL * * if flags & PyBUF_STRIDES: # <<<<<<<<<<<<<< @@ -13553,7 +13616,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu goto __pyx_L7; } - /* "View.MemoryView":530 + /* "View.MemoryView":532 * info.strides = self.view.strides * else: * info.strides = NULL # <<<<<<<<<<<<<< @@ -13565,7 +13628,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu } __pyx_L7:; - /* "View.MemoryView":532 + /* "View.MemoryView":534 * info.strides = NULL * * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< @@ -13575,7 +13638,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_1 = ((__pyx_v_flags & PyBUF_INDIRECT) != 0); if (__pyx_t_1) { - /* "View.MemoryView":533 + /* "View.MemoryView":535 * * if flags & PyBUF_INDIRECT: * info.suboffsets = self.view.suboffsets # <<<<<<<<<<<<<< @@ -13585,7 +13648,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_4 = __pyx_v_self->view.suboffsets; __pyx_v_info->suboffsets = __pyx_t_4; - /* "View.MemoryView":532 + /* "View.MemoryView":534 * info.strides = NULL * * if flags & PyBUF_INDIRECT: # <<<<<<<<<<<<<< @@ -13595,7 +13658,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu goto __pyx_L8; } - /* "View.MemoryView":535 + /* "View.MemoryView":537 * info.suboffsets = self.view.suboffsets * else: * info.suboffsets = NULL # <<<<<<<<<<<<<< @@ -13607,7 +13670,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu } __pyx_L8:; - /* "View.MemoryView":537 + /* "View.MemoryView":539 * info.suboffsets = NULL * * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< @@ -13617,7 +13680,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_1 = ((__pyx_v_flags & PyBUF_FORMAT) != 0); if (__pyx_t_1) { - /* "View.MemoryView":538 + /* "View.MemoryView":540 * * if flags & PyBUF_FORMAT: * info.format = self.view.format # <<<<<<<<<<<<<< @@ -13627,7 +13690,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_5 = __pyx_v_self->view.format; __pyx_v_info->format = __pyx_t_5; - /* "View.MemoryView":537 + /* "View.MemoryView":539 * info.suboffsets = NULL * * if flags & PyBUF_FORMAT: # <<<<<<<<<<<<<< @@ -13637,7 +13700,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu goto __pyx_L9; } - /* "View.MemoryView":540 + /* "View.MemoryView":542 * info.format = self.view.format * else: * info.format = NULL # <<<<<<<<<<<<<< @@ -13649,7 +13712,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu } __pyx_L9:; - /* "View.MemoryView":542 + /* "View.MemoryView":544 * info.format = NULL * * info.buf = self.view.buf # <<<<<<<<<<<<<< @@ -13659,7 +13722,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_6 = __pyx_v_self->view.buf; __pyx_v_info->buf = __pyx_t_6; - /* "View.MemoryView":543 + /* "View.MemoryView":545 * * info.buf = self.view.buf * info.ndim = self.view.ndim # <<<<<<<<<<<<<< @@ -13669,7 +13732,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_7 = __pyx_v_self->view.ndim; __pyx_v_info->ndim = __pyx_t_7; - /* "View.MemoryView":544 + /* "View.MemoryView":546 * info.buf = self.view.buf * info.ndim = self.view.ndim * info.itemsize = self.view.itemsize # <<<<<<<<<<<<<< @@ -13679,7 +13742,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_8 = __pyx_v_self->view.itemsize; __pyx_v_info->itemsize = __pyx_t_8; - /* "View.MemoryView":545 + /* "View.MemoryView":547 * info.ndim = self.view.ndim * info.itemsize = self.view.itemsize * info.len = self.view.len # <<<<<<<<<<<<<< @@ -13689,7 +13752,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_8 = __pyx_v_self->view.len; __pyx_v_info->len = __pyx_t_8; - /* "View.MemoryView":546 + /* "View.MemoryView":548 * info.itemsize = self.view.itemsize * info.len = self.view.len * info.readonly = self.view.readonly # <<<<<<<<<<<<<< @@ -13699,7 +13762,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __pyx_t_1 = __pyx_v_self->view.readonly; __pyx_v_info->readonly = __pyx_t_1; - /* "View.MemoryView":547 + /* "View.MemoryView":549 * info.len = self.view.len * info.readonly = self.view.readonly * info.obj = self # <<<<<<<<<<<<<< @@ -13712,7 +13775,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu __Pyx_DECREF(__pyx_v_info->obj); __pyx_v_info->obj = ((PyObject *)__pyx_v_self); - /* "View.MemoryView":518 + /* "View.MemoryView":520 * * @cname('getbuffer') * def __getbuffer__(self, Py_buffer *info, int flags): # <<<<<<<<<<<<<< @@ -13742,7 +13805,7 @@ static int __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_8__getbu return __pyx_r; } -/* "View.MemoryView":553 +/* "View.MemoryView":555 * * @property * def T(self): # <<<<<<<<<<<<<< @@ -13774,29 +13837,29 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct _ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":554 + /* "View.MemoryView":556 * @property * def T(self): * cdef _memoryviewslice result = memoryview_copy(self) # <<<<<<<<<<<<<< * transpose_memslice(&result.from_slice) * return result */ - __pyx_t_1 = __pyx_memoryview_copy_object(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 554, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_copy_object(__pyx_v_self); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 556, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_memoryviewslice_type))))) __PYX_ERR(1, 554, __pyx_L1_error) + if (!(likely(((__pyx_t_1) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_1, __pyx_memoryviewslice_type))))) __PYX_ERR(1, 556, __pyx_L1_error) __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_1); __pyx_t_1 = 0; - /* "View.MemoryView":555 + /* "View.MemoryView":557 * def T(self): * cdef _memoryviewslice result = memoryview_copy(self) * transpose_memslice(&result.from_slice) # <<<<<<<<<<<<<< * return result * */ - __pyx_t_2 = __pyx_memslice_transpose((&__pyx_v_result->from_slice)); if (unlikely(__pyx_t_2 == ((int)0))) __PYX_ERR(1, 555, __pyx_L1_error) + __pyx_t_2 = __pyx_memslice_transpose((&__pyx_v_result->from_slice)); if (unlikely(__pyx_t_2 == ((int)0))) __PYX_ERR(1, 557, __pyx_L1_error) - /* "View.MemoryView":556 + /* "View.MemoryView":558 * cdef _memoryviewslice result = memoryview_copy(self) * transpose_memslice(&result.from_slice) * return result # <<<<<<<<<<<<<< @@ -13808,7 +13871,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct _ __pyx_r = ((PyObject *)__pyx_v_result); goto __pyx_L0; - /* "View.MemoryView":553 + /* "View.MemoryView":555 * * @property * def T(self): # <<<<<<<<<<<<<< @@ -13828,7 +13891,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_1T___get__(struct _ return __pyx_r; } -/* "View.MemoryView":559 +/* "View.MemoryView":561 * * @property * def base(self): # <<<<<<<<<<<<<< @@ -13854,7 +13917,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struc __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":560 + /* "View.MemoryView":562 * @property * def base(self): * return self.obj # <<<<<<<<<<<<<< @@ -13866,7 +13929,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struc __pyx_r = __pyx_v_self->obj; goto __pyx_L0; - /* "View.MemoryView":559 + /* "View.MemoryView":561 * * @property * def base(self): # <<<<<<<<<<<<<< @@ -13881,7 +13944,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4base___get__(struc return __pyx_r; } -/* "View.MemoryView":563 +/* "View.MemoryView":565 * * @property * def shape(self): # <<<<<<<<<<<<<< @@ -13916,7 +13979,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(stru int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":564 + /* "View.MemoryView":566 * @property * def shape(self): * return tuple([length for length in self.view.shape[:self.view.ndim]]) # <<<<<<<<<<<<<< @@ -13924,25 +13987,25 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(stru * @property */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 564, __pyx_L1_error) + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 566, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); for (__pyx_t_4 = __pyx_v_self->view.shape; __pyx_t_4 < __pyx_t_3; __pyx_t_4++) { __pyx_t_2 = __pyx_t_4; __pyx_v_length = (__pyx_t_2[0]); - __pyx_t_5 = PyInt_FromSsize_t(__pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 564, __pyx_L1_error) + __pyx_t_5 = PyInt_FromSsize_t(__pyx_v_length); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 566, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); - if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(1, 564, __pyx_L1_error) + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(1, 566, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } - __pyx_t_5 = PyList_AsTuple(((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 564, __pyx_L1_error) + __pyx_t_5 = PyList_AsTuple(((PyObject*)__pyx_t_1)); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 566, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_5; __pyx_t_5 = 0; goto __pyx_L0; - /* "View.MemoryView":563 + /* "View.MemoryView":565 * * @property * def shape(self): # <<<<<<<<<<<<<< @@ -13962,7 +14025,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_5shape___get__(stru return __pyx_r; } -/* "View.MemoryView":567 +/* "View.MemoryView":569 * * @property * def strides(self): # <<<<<<<<<<<<<< @@ -13998,7 +14061,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(st int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":568 + /* "View.MemoryView":570 * @property * def strides(self): * if self.view.strides == NULL: # <<<<<<<<<<<<<< @@ -14008,20 +14071,20 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(st __pyx_t_1 = ((__pyx_v_self->view.strides == NULL) != 0); if (unlikely(__pyx_t_1)) { - /* "View.MemoryView":570 + /* "View.MemoryView":572 * if self.view.strides == NULL: * * raise ValueError("Buffer view does not expose strides") # <<<<<<<<<<<<<< * * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) */ - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__26, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 570, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__26, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 572, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 570, __pyx_L1_error) + __PYX_ERR(1, 572, __pyx_L1_error) - /* "View.MemoryView":568 + /* "View.MemoryView":570 * @property * def strides(self): * if self.view.strides == NULL: # <<<<<<<<<<<<<< @@ -14030,7 +14093,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(st */ } - /* "View.MemoryView":572 + /* "View.MemoryView":574 * raise ValueError("Buffer view does not expose strides") * * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) # <<<<<<<<<<<<<< @@ -14038,25 +14101,25 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(st * @property */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 572, __pyx_L1_error) + __pyx_t_2 = PyList_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 574, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_4 = (__pyx_v_self->view.strides + __pyx_v_self->view.ndim); for (__pyx_t_5 = __pyx_v_self->view.strides; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { __pyx_t_3 = __pyx_t_5; __pyx_v_stride = (__pyx_t_3[0]); - __pyx_t_6 = PyInt_FromSsize_t(__pyx_v_stride); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 572, __pyx_L1_error) + __pyx_t_6 = PyInt_FromSsize_t(__pyx_v_stride); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 574, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); - if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 572, __pyx_L1_error) + if (unlikely(__Pyx_ListComp_Append(__pyx_t_2, (PyObject*)__pyx_t_6))) __PYX_ERR(1, 574, __pyx_L1_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } - __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 572, __pyx_L1_error) + __pyx_t_6 = PyList_AsTuple(((PyObject*)__pyx_t_2)); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 574, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_6; __pyx_t_6 = 0; goto __pyx_L0; - /* "View.MemoryView":567 + /* "View.MemoryView":569 * * @property * def strides(self): # <<<<<<<<<<<<<< @@ -14076,7 +14139,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_7strides___get__(st return __pyx_r; } -/* "View.MemoryView":575 +/* "View.MemoryView":577 * * @property * def suboffsets(self): # <<<<<<<<<<<<<< @@ -14112,7 +14175,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":576 + /* "View.MemoryView":578 * @property * def suboffsets(self): * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< @@ -14122,7 +14185,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get_ __pyx_t_1 = ((__pyx_v_self->view.suboffsets == NULL) != 0); if (__pyx_t_1) { - /* "View.MemoryView":577 + /* "View.MemoryView":579 * def suboffsets(self): * if self.view.suboffsets == NULL: * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< @@ -14130,16 +14193,16 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get_ * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 577, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_v_self->view.ndim); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 579, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyNumber_Multiply(__pyx_tuple__27, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 577, __pyx_L1_error) + __pyx_t_3 = PyNumber_Multiply(__pyx_tuple__27, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 579, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "View.MemoryView":576 + /* "View.MemoryView":578 * @property * def suboffsets(self): * if self.view.suboffsets == NULL: # <<<<<<<<<<<<<< @@ -14148,7 +14211,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get_ */ } - /* "View.MemoryView":579 + /* "View.MemoryView":581 * return (-1,) * self.view.ndim * * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) # <<<<<<<<<<<<<< @@ -14156,25 +14219,25 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get_ * @property */ __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 579, __pyx_L1_error) + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 581, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_5 = (__pyx_v_self->view.suboffsets + __pyx_v_self->view.ndim); for (__pyx_t_6 = __pyx_v_self->view.suboffsets; __pyx_t_6 < __pyx_t_5; __pyx_t_6++) { __pyx_t_4 = __pyx_t_6; __pyx_v_suboffset = (__pyx_t_4[0]); - __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_suboffset); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 579, __pyx_L1_error) + __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_suboffset); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 581, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (unlikely(__Pyx_ListComp_Append(__pyx_t_3, (PyObject*)__pyx_t_2))) __PYX_ERR(1, 579, __pyx_L1_error) + if (unlikely(__Pyx_ListComp_Append(__pyx_t_3, (PyObject*)__pyx_t_2))) __PYX_ERR(1, 581, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } - __pyx_t_2 = PyList_AsTuple(((PyObject*)__pyx_t_3)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 579, __pyx_L1_error) + __pyx_t_2 = PyList_AsTuple(((PyObject*)__pyx_t_3)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 581, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":575 + /* "View.MemoryView":577 * * @property * def suboffsets(self): # <<<<<<<<<<<<<< @@ -14194,7 +14257,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_10suboffsets___get_ return __pyx_r; } -/* "View.MemoryView":582 +/* "View.MemoryView":584 * * @property * def ndim(self): # <<<<<<<<<<<<<< @@ -14224,7 +14287,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struc int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":583 + /* "View.MemoryView":585 * @property * def ndim(self): * return self.view.ndim # <<<<<<<<<<<<<< @@ -14232,13 +14295,13 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struc * @property */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 583, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_self->view.ndim); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 585, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "View.MemoryView":582 + /* "View.MemoryView":584 * * @property * def ndim(self): # <<<<<<<<<<<<<< @@ -14257,7 +14320,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4ndim___get__(struc return __pyx_r; } -/* "View.MemoryView":586 +/* "View.MemoryView":588 * * @property * def itemsize(self): # <<<<<<<<<<<<<< @@ -14287,7 +14350,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(s int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":587 + /* "View.MemoryView":589 * @property * def itemsize(self): * return self.view.itemsize # <<<<<<<<<<<<<< @@ -14295,13 +14358,13 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(s * @property */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyInt_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 587, __pyx_L1_error) + __pyx_t_1 = PyInt_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 589, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "View.MemoryView":586 + /* "View.MemoryView":588 * * @property * def itemsize(self): # <<<<<<<<<<<<<< @@ -14320,7 +14383,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_8itemsize___get__(s return __pyx_r; } -/* "View.MemoryView":590 +/* "View.MemoryView":592 * * @property * def nbytes(self): # <<<<<<<<<<<<<< @@ -14352,7 +14415,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(str int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":591 + /* "View.MemoryView":593 * @property * def nbytes(self): * return self.size * self.view.itemsize # <<<<<<<<<<<<<< @@ -14360,11 +14423,11 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(str * @property */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_size); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 591, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_size); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 593, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 591, __pyx_L1_error) + __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_self->view.itemsize); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 593, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyNumber_Multiply(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 591, __pyx_L1_error) + __pyx_t_3 = PyNumber_Multiply(__pyx_t_1, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 593, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -14372,7 +14435,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(str __pyx_t_3 = 0; goto __pyx_L0; - /* "View.MemoryView":590 + /* "View.MemoryView":592 * * @property * def nbytes(self): # <<<<<<<<<<<<<< @@ -14393,7 +14456,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_6nbytes___get__(str return __pyx_r; } -/* "View.MemoryView":594 +/* "View.MemoryView":596 * * @property * def size(self): # <<<<<<<<<<<<<< @@ -14430,7 +14493,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struc int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":595 + /* "View.MemoryView":597 * @property * def size(self): * if self._size is None: # <<<<<<<<<<<<<< @@ -14441,7 +14504,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struc __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "View.MemoryView":596 + /* "View.MemoryView":598 * def size(self): * if self._size is None: * result = 1 # <<<<<<<<<<<<<< @@ -14451,7 +14514,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struc __Pyx_INCREF(__pyx_int_1); __pyx_v_result = __pyx_int_1; - /* "View.MemoryView":598 + /* "View.MemoryView":600 * result = 1 * * for length in self.view.shape[:self.view.ndim]: # <<<<<<<<<<<<<< @@ -14461,25 +14524,25 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struc __pyx_t_4 = (__pyx_v_self->view.shape + __pyx_v_self->view.ndim); for (__pyx_t_5 = __pyx_v_self->view.shape; __pyx_t_5 < __pyx_t_4; __pyx_t_5++) { __pyx_t_3 = __pyx_t_5; - __pyx_t_6 = PyInt_FromSsize_t((__pyx_t_3[0])); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 598, __pyx_L1_error) + __pyx_t_6 = PyInt_FromSsize_t((__pyx_t_3[0])); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 600, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_6); __pyx_t_6 = 0; - /* "View.MemoryView":599 + /* "View.MemoryView":601 * * for length in self.view.shape[:self.view.ndim]: * result *= length # <<<<<<<<<<<<<< * * self._size = result */ - __pyx_t_6 = PyNumber_InPlaceMultiply(__pyx_v_result, __pyx_v_length); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 599, __pyx_L1_error) + __pyx_t_6 = PyNumber_InPlaceMultiply(__pyx_v_result, __pyx_v_length); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 601, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF_SET(__pyx_v_result, __pyx_t_6); __pyx_t_6 = 0; } - /* "View.MemoryView":601 + /* "View.MemoryView":603 * result *= length * * self._size = result # <<<<<<<<<<<<<< @@ -14492,7 +14555,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struc __Pyx_DECREF(__pyx_v_self->_size); __pyx_v_self->_size = __pyx_v_result; - /* "View.MemoryView":595 + /* "View.MemoryView":597 * @property * def size(self): * if self._size is None: # <<<<<<<<<<<<<< @@ -14501,7 +14564,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struc */ } - /* "View.MemoryView":603 + /* "View.MemoryView":605 * self._size = result * * return self._size # <<<<<<<<<<<<<< @@ -14513,7 +14576,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struc __pyx_r = __pyx_v_self->_size; goto __pyx_L0; - /* "View.MemoryView":594 + /* "View.MemoryView":596 * * @property * def size(self): # <<<<<<<<<<<<<< @@ -14534,7 +14597,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_10memoryview_4size___get__(struc return __pyx_r; } -/* "View.MemoryView":605 +/* "View.MemoryView":607 * return self._size * * def __len__(self): # <<<<<<<<<<<<<< @@ -14561,7 +14624,7 @@ static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_1 int __pyx_t_1; __Pyx_RefNannySetupContext("__len__", 0); - /* "View.MemoryView":606 + /* "View.MemoryView":608 * * def __len__(self): * if self.view.ndim >= 1: # <<<<<<<<<<<<<< @@ -14571,7 +14634,7 @@ static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_1 __pyx_t_1 = ((__pyx_v_self->view.ndim >= 1) != 0); if (__pyx_t_1) { - /* "View.MemoryView":607 + /* "View.MemoryView":609 * def __len__(self): * if self.view.ndim >= 1: * return self.view.shape[0] # <<<<<<<<<<<<<< @@ -14581,7 +14644,7 @@ static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_1 __pyx_r = (__pyx_v_self->view.shape[0]); goto __pyx_L0; - /* "View.MemoryView":606 + /* "View.MemoryView":608 * * def __len__(self): * if self.view.ndim >= 1: # <<<<<<<<<<<<<< @@ -14590,7 +14653,7 @@ static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_1 */ } - /* "View.MemoryView":609 + /* "View.MemoryView":611 * return self.view.shape[0] * * return 0 # <<<<<<<<<<<<<< @@ -14600,7 +14663,7 @@ static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_1 __pyx_r = 0; goto __pyx_L0; - /* "View.MemoryView":605 + /* "View.MemoryView":607 * return self._size * * def __len__(self): # <<<<<<<<<<<<<< @@ -14614,7 +14677,7 @@ static Py_ssize_t __pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_1 return __pyx_r; } -/* "View.MemoryView":611 +/* "View.MemoryView":613 * return 0 * * def __repr__(self): # <<<<<<<<<<<<<< @@ -14646,7 +14709,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__repr__", 0); - /* "View.MemoryView":612 + /* "View.MemoryView":614 * * def __repr__(self): * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< @@ -14654,33 +14717,33 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12 * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 612, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 614, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 612, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 614, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 612, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 614, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "View.MemoryView":613 + /* "View.MemoryView":615 * def __repr__(self): * return "" % (self.base.__class__.__name__, * id(self)) # <<<<<<<<<<<<<< * * def __str__(self): */ - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_builtin_id, ((PyObject *)__pyx_v_self)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 613, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_builtin_id, ((PyObject *)__pyx_v_self)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 615, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - /* "View.MemoryView":612 + /* "View.MemoryView":614 * * def __repr__(self): * return "" % (self.base.__class__.__name__, # <<<<<<<<<<<<<< * id(self)) * */ - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 612, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 614, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -14688,14 +14751,14 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12 PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_MemoryView_of_r_at_0x_x, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 612, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_MemoryView_of_r_at_0x_x, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 614, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":611 + /* "View.MemoryView":613 * return 0 * * def __repr__(self): # <<<<<<<<<<<<<< @@ -14716,7 +14779,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_12 return __pyx_r; } -/* "View.MemoryView":615 +/* "View.MemoryView":617 * id(self)) * * def __str__(self): # <<<<<<<<<<<<<< @@ -14747,7 +14810,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__str__", 0); - /* "View.MemoryView":616 + /* "View.MemoryView":618 * * def __str__(self): * return "" % (self.base.__class__.__name__,) # <<<<<<<<<<<<<< @@ -14755,27 +14818,27 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14 * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 616, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_self), __pyx_n_s_base); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 616, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_class); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 616, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_name_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 616, __pyx_L1_error) + __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 618, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_MemoryView_of_r_object, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 616, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_MemoryView_of_r_object, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 618, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "View.MemoryView":615 + /* "View.MemoryView":617 * id(self)) * * def __str__(self): # <<<<<<<<<<<<<< @@ -14795,7 +14858,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_14 return __pyx_r; } -/* "View.MemoryView":619 +/* "View.MemoryView":621 * * * def is_c_contig(self): # <<<<<<<<<<<<<< @@ -14828,17 +14891,17 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("is_c_contig", 0); - /* "View.MemoryView":622 + /* "View.MemoryView":624 * cdef __Pyx_memviewslice *mslice * cdef __Pyx_memviewslice tmp * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< * return slice_is_contig(mslice[0], 'C', self.view.ndim) * */ - __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 622, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 624, __pyx_L1_error) __pyx_v_mslice = __pyx_t_1; - /* "View.MemoryView":623 + /* "View.MemoryView":625 * cdef __Pyx_memviewslice tmp * mslice = get_slice_from_memview(self, &tmp) * return slice_is_contig(mslice[0], 'C', self.view.ndim) # <<<<<<<<<<<<<< @@ -14846,13 +14909,13 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16 * def is_f_contig(self): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'C', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 623, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'C', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 625, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":619 + /* "View.MemoryView":621 * * * def is_c_contig(self): # <<<<<<<<<<<<<< @@ -14871,7 +14934,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_16 return __pyx_r; } -/* "View.MemoryView":625 +/* "View.MemoryView":627 * return slice_is_contig(mslice[0], 'C', self.view.ndim) * * def is_f_contig(self): # <<<<<<<<<<<<<< @@ -14904,17 +14967,17 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("is_f_contig", 0); - /* "View.MemoryView":628 + /* "View.MemoryView":630 * cdef __Pyx_memviewslice *mslice * cdef __Pyx_memviewslice tmp * mslice = get_slice_from_memview(self, &tmp) # <<<<<<<<<<<<<< * return slice_is_contig(mslice[0], 'F', self.view.ndim) * */ - __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 628, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_get_slice_from_memoryview(__pyx_v_self, (&__pyx_v_tmp)); if (unlikely(__pyx_t_1 == ((__Pyx_memviewslice *)NULL))) __PYX_ERR(1, 630, __pyx_L1_error) __pyx_v_mslice = __pyx_t_1; - /* "View.MemoryView":629 + /* "View.MemoryView":631 * cdef __Pyx_memviewslice tmp * mslice = get_slice_from_memview(self, &tmp) * return slice_is_contig(mslice[0], 'F', self.view.ndim) # <<<<<<<<<<<<<< @@ -14922,13 +14985,13 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18 * def copy(self): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'F', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 629, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_memviewslice_is_contig((__pyx_v_mslice[0]), 'F', __pyx_v_self->view.ndim)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 631, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":625 + /* "View.MemoryView":627 * return slice_is_contig(mslice[0], 'C', self.view.ndim) * * def is_f_contig(self): # <<<<<<<<<<<<<< @@ -14947,7 +15010,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_18 return __pyx_r; } -/* "View.MemoryView":631 +/* "View.MemoryView":633 * return slice_is_contig(mslice[0], 'F', self.view.ndim) * * def copy(self): # <<<<<<<<<<<<<< @@ -14980,7 +15043,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("copy", 0); - /* "View.MemoryView":633 + /* "View.MemoryView":635 * def copy(self): * cdef __Pyx_memviewslice mslice * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS # <<<<<<<<<<<<<< @@ -14989,7 +15052,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20 */ __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_F_CONTIGUOUS)); - /* "View.MemoryView":635 + /* "View.MemoryView":637 * cdef int flags = self.flags & ~PyBUF_F_CONTIGUOUS * * slice_copy(self, &mslice) # <<<<<<<<<<<<<< @@ -14998,17 +15061,17 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20 */ __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_mslice)); - /* "View.MemoryView":636 + /* "View.MemoryView":638 * * slice_copy(self, &mslice) * mslice = slice_copy_contig(&mslice, "c", self.view.ndim, # <<<<<<<<<<<<<< * self.view.itemsize, * flags|PyBUF_C_CONTIGUOUS, */ - __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_mslice), ((char *)"c"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_C_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 636, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_mslice), ((char *)"c"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_C_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 638, __pyx_L1_error) __pyx_v_mslice = __pyx_t_1; - /* "View.MemoryView":641 + /* "View.MemoryView":643 * self.dtype_is_object) * * return memoryview_copy_from_slice(self, &mslice) # <<<<<<<<<<<<<< @@ -15016,13 +15079,13 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20 * def copy_fortran(self): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_mslice)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 641, __pyx_L1_error) + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_mslice)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 643, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":631 + /* "View.MemoryView":633 * return slice_is_contig(mslice[0], 'F', self.view.ndim) * * def copy(self): # <<<<<<<<<<<<<< @@ -15041,7 +15104,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_20 return __pyx_r; } -/* "View.MemoryView":643 +/* "View.MemoryView":645 * return memoryview_copy_from_slice(self, &mslice) * * def copy_fortran(self): # <<<<<<<<<<<<<< @@ -15075,7 +15138,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("copy_fortran", 0); - /* "View.MemoryView":645 + /* "View.MemoryView":647 * def copy_fortran(self): * cdef __Pyx_memviewslice src, dst * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS # <<<<<<<<<<<<<< @@ -15084,7 +15147,7 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22 */ __pyx_v_flags = (__pyx_v_self->flags & (~PyBUF_C_CONTIGUOUS)); - /* "View.MemoryView":647 + /* "View.MemoryView":649 * cdef int flags = self.flags & ~PyBUF_C_CONTIGUOUS * * slice_copy(self, &src) # <<<<<<<<<<<<<< @@ -15093,17 +15156,17 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22 */ __pyx_memoryview_slice_copy(__pyx_v_self, (&__pyx_v_src)); - /* "View.MemoryView":648 + /* "View.MemoryView":650 * * slice_copy(self, &src) * dst = slice_copy_contig(&src, "fortran", self.view.ndim, # <<<<<<<<<<<<<< * self.view.itemsize, * flags|PyBUF_F_CONTIGUOUS, */ - __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_src), ((char *)"fortran"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_F_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 648, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_copy_new_contig((&__pyx_v_src), ((char *)"fortran"), __pyx_v_self->view.ndim, __pyx_v_self->view.itemsize, (__pyx_v_flags | PyBUF_F_CONTIGUOUS), __pyx_v_self->dtype_is_object); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 650, __pyx_L1_error) __pyx_v_dst = __pyx_t_1; - /* "View.MemoryView":653 + /* "View.MemoryView":655 * self.dtype_is_object) * * return memoryview_copy_from_slice(self, &dst) # <<<<<<<<<<<<<< @@ -15111,13 +15174,13 @@ static PyObject *__pyx_memoryview___pyx_pf_15View_dot_MemoryView_10memoryview_22 * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_dst)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 653, __pyx_L1_error) + __pyx_t_2 = __pyx_memoryview_copy_object_from_slice(__pyx_v_self, (&__pyx_v_dst)); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 655, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":643 + /* "View.MemoryView":645 * return memoryview_copy_from_slice(self, &mslice) * * def copy_fortran(self): # <<<<<<<<<<<<<< @@ -15249,7 +15312,7 @@ static PyObject *__pyx_pf___pyx_memoryview_2__setstate_cython__(CYTHON_UNUSED st return __pyx_r; } -/* "View.MemoryView":657 +/* "View.MemoryView":659 * * @cname('__pyx_memoryview_new') * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, __Pyx_TypeInfo *typeinfo): # <<<<<<<<<<<<<< @@ -15269,18 +15332,18 @@ static PyObject *__pyx_memoryview_new(PyObject *__pyx_v_o, int __pyx_v_flags, in int __pyx_clineno = 0; __Pyx_RefNannySetupContext("memoryview_cwrapper", 0); - /* "View.MemoryView":658 + /* "View.MemoryView":660 * @cname('__pyx_memoryview_new') * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, __Pyx_TypeInfo *typeinfo): * cdef memoryview result = memoryview(o, flags, dtype_is_object) # <<<<<<<<<<<<<< * result.typeinfo = typeinfo * return result */ - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 658, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_flags); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 660, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 658, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 660, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 658, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 660, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_v_o); __Pyx_GIVEREF(__pyx_v_o); @@ -15291,13 +15354,13 @@ static PyObject *__pyx_memoryview_new(PyObject *__pyx_v_o, int __pyx_v_flags, in PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_memoryview_type), __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 658, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_memoryview_type), __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 660, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_v_result = ((struct __pyx_memoryview_obj *)__pyx_t_2); __pyx_t_2 = 0; - /* "View.MemoryView":659 + /* "View.MemoryView":661 * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, __Pyx_TypeInfo *typeinfo): * cdef memoryview result = memoryview(o, flags, dtype_is_object) * result.typeinfo = typeinfo # <<<<<<<<<<<<<< @@ -15306,7 +15369,7 @@ static PyObject *__pyx_memoryview_new(PyObject *__pyx_v_o, int __pyx_v_flags, in */ __pyx_v_result->typeinfo = __pyx_v_typeinfo; - /* "View.MemoryView":660 + /* "View.MemoryView":662 * cdef memoryview result = memoryview(o, flags, dtype_is_object) * result.typeinfo = typeinfo * return result # <<<<<<<<<<<<<< @@ -15318,7 +15381,7 @@ static PyObject *__pyx_memoryview_new(PyObject *__pyx_v_o, int __pyx_v_flags, in __pyx_r = ((PyObject *)__pyx_v_result); goto __pyx_L0; - /* "View.MemoryView":657 + /* "View.MemoryView":659 * * @cname('__pyx_memoryview_new') * cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, __Pyx_TypeInfo *typeinfo): # <<<<<<<<<<<<<< @@ -15340,7 +15403,7 @@ static PyObject *__pyx_memoryview_new(PyObject *__pyx_v_o, int __pyx_v_flags, in return __pyx_r; } -/* "View.MemoryView":663 +/* "View.MemoryView":665 * * @cname('__pyx_memoryview_check') * cdef inline bint memoryview_check(object o): # <<<<<<<<<<<<<< @@ -15354,7 +15417,7 @@ static CYTHON_INLINE int __pyx_memoryview_check(PyObject *__pyx_v_o) { int __pyx_t_1; __Pyx_RefNannySetupContext("memoryview_check", 0); - /* "View.MemoryView":664 + /* "View.MemoryView":666 * @cname('__pyx_memoryview_check') * cdef inline bint memoryview_check(object o): * return isinstance(o, memoryview) # <<<<<<<<<<<<<< @@ -15365,7 +15428,7 @@ static CYTHON_INLINE int __pyx_memoryview_check(PyObject *__pyx_v_o) { __pyx_r = __pyx_t_1; goto __pyx_L0; - /* "View.MemoryView":663 + /* "View.MemoryView":665 * * @cname('__pyx_memoryview_check') * cdef inline bint memoryview_check(object o): # <<<<<<<<<<<<<< @@ -15379,7 +15442,7 @@ static CYTHON_INLINE int __pyx_memoryview_check(PyObject *__pyx_v_o) { return __pyx_r; } -/* "View.MemoryView":666 +/* "View.MemoryView":668 * return isinstance(o, memoryview) * * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< @@ -15413,7 +15476,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { int __pyx_clineno = 0; __Pyx_RefNannySetupContext("_unellipsify", 0); - /* "View.MemoryView":671 + /* "View.MemoryView":673 * full slices. * """ * if not isinstance(index, tuple): # <<<<<<<<<<<<<< @@ -15424,14 +15487,14 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_t_2 = ((!(__pyx_t_1 != 0)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":672 + /* "View.MemoryView":674 * """ * if not isinstance(index, tuple): * tup = (index,) # <<<<<<<<<<<<<< * else: * tup = index */ - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 672, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 674, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_v_index); __Pyx_GIVEREF(__pyx_v_index); @@ -15439,7 +15502,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_v_tup = __pyx_t_3; __pyx_t_3 = 0; - /* "View.MemoryView":671 + /* "View.MemoryView":673 * full slices. * """ * if not isinstance(index, tuple): # <<<<<<<<<<<<<< @@ -15449,7 +15512,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { goto __pyx_L3; } - /* "View.MemoryView":674 + /* "View.MemoryView":676 * tup = (index,) * else: * tup = index # <<<<<<<<<<<<<< @@ -15462,19 +15525,19 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { } __pyx_L3:; - /* "View.MemoryView":676 + /* "View.MemoryView":678 * tup = index * * result = [] # <<<<<<<<<<<<<< * have_slices = False * seen_ellipsis = False */ - __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 676, __pyx_L1_error) + __pyx_t_3 = PyList_New(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 678, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_v_result = ((PyObject*)__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":677 + /* "View.MemoryView":679 * * result = [] * have_slices = False # <<<<<<<<<<<<<< @@ -15483,7 +15546,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { */ __pyx_v_have_slices = 0; - /* "View.MemoryView":678 + /* "View.MemoryView":680 * result = [] * have_slices = False * seen_ellipsis = False # <<<<<<<<<<<<<< @@ -15492,7 +15555,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { */ __pyx_v_seen_ellipsis = 0; - /* "View.MemoryView":679 + /* "View.MemoryView":681 * have_slices = False * seen_ellipsis = False * for idx, item in enumerate(tup): # <<<<<<<<<<<<<< @@ -15505,26 +15568,26 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_t_4 = __pyx_v_tup; __Pyx_INCREF(__pyx_t_4); __pyx_t_5 = 0; __pyx_t_6 = NULL; } else { - __pyx_t_5 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_v_tup); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 679, __pyx_L1_error) + __pyx_t_5 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_v_tup); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 681, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_6 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 679, __pyx_L1_error) + __pyx_t_6 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 681, __pyx_L1_error) } for (;;) { if (likely(!__pyx_t_6)) { if (likely(PyList_CheckExact(__pyx_t_4))) { if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_7 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_5); __Pyx_INCREF(__pyx_t_7); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(1, 679, __pyx_L1_error) + __pyx_t_7 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_5); __Pyx_INCREF(__pyx_t_7); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(1, 681, __pyx_L1_error) #else - __pyx_t_7 = PySequence_ITEM(__pyx_t_4, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 679, __pyx_L1_error) + __pyx_t_7 = PySequence_ITEM(__pyx_t_4, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 681, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); #endif } else { if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_4)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_5); __Pyx_INCREF(__pyx_t_7); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(1, 679, __pyx_L1_error) + __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_5); __Pyx_INCREF(__pyx_t_7); __pyx_t_5++; if (unlikely(0 < 0)) __PYX_ERR(1, 681, __pyx_L1_error) #else - __pyx_t_7 = PySequence_ITEM(__pyx_t_4, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 679, __pyx_L1_error) + __pyx_t_7 = PySequence_ITEM(__pyx_t_4, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 681, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); #endif } @@ -15534,7 +15597,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(1, 679, __pyx_L1_error) + else __PYX_ERR(1, 681, __pyx_L1_error) } break; } @@ -15544,13 +15607,13 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_t_7 = 0; __Pyx_INCREF(__pyx_t_3); __Pyx_XDECREF_SET(__pyx_v_idx, __pyx_t_3); - __pyx_t_7 = __Pyx_PyInt_AddObjC(__pyx_t_3, __pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 679, __pyx_L1_error) + __pyx_t_7 = __Pyx_PyInt_AddObjC(__pyx_t_3, __pyx_int_1, 1, 0, 0); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 681, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = __pyx_t_7; __pyx_t_7 = 0; - /* "View.MemoryView":680 + /* "View.MemoryView":682 * seen_ellipsis = False * for idx, item in enumerate(tup): * if item is Ellipsis: # <<<<<<<<<<<<<< @@ -15561,7 +15624,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_t_1 = (__pyx_t_2 != 0); if (__pyx_t_1) { - /* "View.MemoryView":681 + /* "View.MemoryView":683 * for idx, item in enumerate(tup): * if item is Ellipsis: * if not seen_ellipsis: # <<<<<<<<<<<<<< @@ -15571,15 +15634,15 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_t_1 = ((!(__pyx_v_seen_ellipsis != 0)) != 0); if (__pyx_t_1) { - /* "View.MemoryView":682 + /* "View.MemoryView":684 * if item is Ellipsis: * if not seen_ellipsis: * result.extend([slice(None)] * (ndim - len(tup) + 1)) # <<<<<<<<<<<<<< * seen_ellipsis = True * else: */ - __pyx_t_8 = PyObject_Length(__pyx_v_tup); if (unlikely(__pyx_t_8 == ((Py_ssize_t)-1))) __PYX_ERR(1, 682, __pyx_L1_error) - __pyx_t_7 = PyList_New(1 * ((((__pyx_v_ndim - __pyx_t_8) + 1)<0) ? 0:((__pyx_v_ndim - __pyx_t_8) + 1))); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 682, __pyx_L1_error) + __pyx_t_8 = PyObject_Length(__pyx_v_tup); if (unlikely(__pyx_t_8 == ((Py_ssize_t)-1))) __PYX_ERR(1, 684, __pyx_L1_error) + __pyx_t_7 = PyList_New(1 * ((((__pyx_v_ndim - __pyx_t_8) + 1)<0) ? 0:((__pyx_v_ndim - __pyx_t_8) + 1))); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 684, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); { Py_ssize_t __pyx_temp; for (__pyx_temp=0; __pyx_temp < ((__pyx_v_ndim - __pyx_t_8) + 1); __pyx_temp++) { @@ -15588,10 +15651,10 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { PyList_SET_ITEM(__pyx_t_7, __pyx_temp, __pyx_slice__30); } } - __pyx_t_9 = __Pyx_PyList_Extend(__pyx_v_result, __pyx_t_7); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 682, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyList_Extend(__pyx_v_result, __pyx_t_7); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 684, __pyx_L1_error) __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - /* "View.MemoryView":683 + /* "View.MemoryView":685 * if not seen_ellipsis: * result.extend([slice(None)] * (ndim - len(tup) + 1)) * seen_ellipsis = True # <<<<<<<<<<<<<< @@ -15600,7 +15663,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { */ __pyx_v_seen_ellipsis = 1; - /* "View.MemoryView":681 + /* "View.MemoryView":683 * for idx, item in enumerate(tup): * if item is Ellipsis: * if not seen_ellipsis: # <<<<<<<<<<<<<< @@ -15610,7 +15673,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { goto __pyx_L7; } - /* "View.MemoryView":685 + /* "View.MemoryView":687 * seen_ellipsis = True * else: * result.append(slice(None)) # <<<<<<<<<<<<<< @@ -15618,11 +15681,11 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { * else: */ /*else*/ { - __pyx_t_9 = __Pyx_PyList_Append(__pyx_v_result, __pyx_slice__30); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 685, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyList_Append(__pyx_v_result, __pyx_slice__30); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 687, __pyx_L1_error) } __pyx_L7:; - /* "View.MemoryView":686 + /* "View.MemoryView":688 * else: * result.append(slice(None)) * have_slices = True # <<<<<<<<<<<<<< @@ -15631,7 +15694,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { */ __pyx_v_have_slices = 1; - /* "View.MemoryView":680 + /* "View.MemoryView":682 * seen_ellipsis = False * for idx, item in enumerate(tup): * if item is Ellipsis: # <<<<<<<<<<<<<< @@ -15641,7 +15704,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { goto __pyx_L6; } - /* "View.MemoryView":688 + /* "View.MemoryView":690 * have_slices = True * else: * if not isinstance(item, slice) and not PyIndex_Check(item): # <<<<<<<<<<<<<< @@ -15661,23 +15724,23 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_L9_bool_binop_done:; if (unlikely(__pyx_t_1)) { - /* "View.MemoryView":689 + /* "View.MemoryView":691 * else: * if not isinstance(item, slice) and not PyIndex_Check(item): * raise TypeError("Cannot index with type '%s'" % type(item)) # <<<<<<<<<<<<<< * * have_slices = have_slices or isinstance(item, slice) */ - __pyx_t_7 = __Pyx_PyString_FormatSafe(__pyx_kp_s_Cannot_index_with_type_s, ((PyObject *)Py_TYPE(__pyx_v_item))); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 689, __pyx_L1_error) + __pyx_t_7 = __Pyx_PyString_FormatSafe(__pyx_kp_s_Cannot_index_with_type_s, ((PyObject *)Py_TYPE(__pyx_v_item))); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 691, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_11 = __Pyx_PyObject_CallOneArg(__pyx_builtin_TypeError, __pyx_t_7); if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 689, __pyx_L1_error) + __pyx_t_11 = __Pyx_PyObject_CallOneArg(__pyx_builtin_TypeError, __pyx_t_7); if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 691, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_11, 0, 0, 0); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __PYX_ERR(1, 689, __pyx_L1_error) + __PYX_ERR(1, 691, __pyx_L1_error) - /* "View.MemoryView":688 + /* "View.MemoryView":690 * have_slices = True * else: * if not isinstance(item, slice) and not PyIndex_Check(item): # <<<<<<<<<<<<<< @@ -15686,7 +15749,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { */ } - /* "View.MemoryView":691 + /* "View.MemoryView":693 * raise TypeError("Cannot index with type '%s'" % type(item)) * * have_slices = have_slices or isinstance(item, slice) # <<<<<<<<<<<<<< @@ -15705,18 +15768,18 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_L11_bool_binop_done:; __pyx_v_have_slices = __pyx_t_1; - /* "View.MemoryView":692 + /* "View.MemoryView":694 * * have_slices = have_slices or isinstance(item, slice) * result.append(item) # <<<<<<<<<<<<<< * * nslices = ndim - len(result) */ - __pyx_t_9 = __Pyx_PyList_Append(__pyx_v_result, __pyx_v_item); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 692, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyList_Append(__pyx_v_result, __pyx_v_item); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 694, __pyx_L1_error) } __pyx_L6:; - /* "View.MemoryView":679 + /* "View.MemoryView":681 * have_slices = False * seen_ellipsis = False * for idx, item in enumerate(tup): # <<<<<<<<<<<<<< @@ -15727,17 +15790,17 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":694 + /* "View.MemoryView":696 * result.append(item) * * nslices = ndim - len(result) # <<<<<<<<<<<<<< * if nslices: * result.extend([slice(None)] * nslices) */ - __pyx_t_5 = PyList_GET_SIZE(__pyx_v_result); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(1, 694, __pyx_L1_error) + __pyx_t_5 = PyList_GET_SIZE(__pyx_v_result); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(1, 696, __pyx_L1_error) __pyx_v_nslices = (__pyx_v_ndim - __pyx_t_5); - /* "View.MemoryView":695 + /* "View.MemoryView":697 * * nslices = ndim - len(result) * if nslices: # <<<<<<<<<<<<<< @@ -15747,14 +15810,14 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_t_1 = (__pyx_v_nslices != 0); if (__pyx_t_1) { - /* "View.MemoryView":696 + /* "View.MemoryView":698 * nslices = ndim - len(result) * if nslices: * result.extend([slice(None)] * nslices) # <<<<<<<<<<<<<< * * return have_slices or nslices, tuple(result) */ - __pyx_t_3 = PyList_New(1 * ((__pyx_v_nslices<0) ? 0:__pyx_v_nslices)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 696, __pyx_L1_error) + __pyx_t_3 = PyList_New(1 * ((__pyx_v_nslices<0) ? 0:__pyx_v_nslices)); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 698, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); { Py_ssize_t __pyx_temp; for (__pyx_temp=0; __pyx_temp < __pyx_v_nslices; __pyx_temp++) { @@ -15763,10 +15826,10 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { PyList_SET_ITEM(__pyx_t_3, __pyx_temp, __pyx_slice__30); } } - __pyx_t_9 = __Pyx_PyList_Extend(__pyx_v_result, __pyx_t_3); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 696, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyList_Extend(__pyx_v_result, __pyx_t_3); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 698, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":695 + /* "View.MemoryView":697 * * nslices = ndim - len(result) * if nslices: # <<<<<<<<<<<<<< @@ -15775,7 +15838,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { */ } - /* "View.MemoryView":698 + /* "View.MemoryView":700 * result.extend([slice(None)] * nslices) * * return have_slices or nslices, tuple(result) # <<<<<<<<<<<<<< @@ -15785,20 +15848,20 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __Pyx_XDECREF(__pyx_r); if (!__pyx_v_have_slices) { } else { - __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_have_slices); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 698, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyBool_FromLong(__pyx_v_have_slices); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 700, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L14_bool_binop_done; } - __pyx_t_4 = PyInt_FromSsize_t(__pyx_v_nslices); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 698, __pyx_L1_error) + __pyx_t_4 = PyInt_FromSsize_t(__pyx_v_nslices); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 700, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_3 = __pyx_t_4; __pyx_t_4 = 0; __pyx_L14_bool_binop_done:; - __pyx_t_4 = PyList_AsTuple(__pyx_v_result); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 698, __pyx_L1_error) + __pyx_t_4 = PyList_AsTuple(__pyx_v_result); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 700, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_11 = PyTuple_New(2); if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 698, __pyx_L1_error) + __pyx_t_11 = PyTuple_New(2); if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 700, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_3); @@ -15810,7 +15873,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { __pyx_t_11 = 0; goto __pyx_L0; - /* "View.MemoryView":666 + /* "View.MemoryView":668 * return isinstance(o, memoryview) * * cdef tuple _unellipsify(object index, int ndim): # <<<<<<<<<<<<<< @@ -15836,7 +15899,7 @@ static PyObject *_unellipsify(PyObject *__pyx_v_index, int __pyx_v_ndim) { return __pyx_r; } -/* "View.MemoryView":700 +/* "View.MemoryView":702 * return have_slices or nslices, tuple(result) * * cdef assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim): # <<<<<<<<<<<<<< @@ -15858,7 +15921,7 @@ static PyObject *assert_direct_dimensions(Py_ssize_t *__pyx_v_suboffsets, int __ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("assert_direct_dimensions", 0); - /* "View.MemoryView":701 + /* "View.MemoryView":703 * * cdef assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim): * for suboffset in suboffsets[:ndim]: # <<<<<<<<<<<<<< @@ -15870,7 +15933,7 @@ static PyObject *assert_direct_dimensions(Py_ssize_t *__pyx_v_suboffsets, int __ __pyx_t_1 = __pyx_t_3; __pyx_v_suboffset = (__pyx_t_1[0]); - /* "View.MemoryView":702 + /* "View.MemoryView":704 * cdef assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim): * for suboffset in suboffsets[:ndim]: * if suboffset >= 0: # <<<<<<<<<<<<<< @@ -15880,20 +15943,20 @@ static PyObject *assert_direct_dimensions(Py_ssize_t *__pyx_v_suboffsets, int __ __pyx_t_4 = ((__pyx_v_suboffset >= 0) != 0); if (unlikely(__pyx_t_4)) { - /* "View.MemoryView":703 + /* "View.MemoryView":705 * for suboffset in suboffsets[:ndim]: * if suboffset >= 0: * raise ValueError("Indirect dimensions not supported") # <<<<<<<<<<<<<< * * */ - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__31, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 703, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__31, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 705, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_Raise(__pyx_t_5, 0, 0, 0); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 703, __pyx_L1_error) + __PYX_ERR(1, 705, __pyx_L1_error) - /* "View.MemoryView":702 + /* "View.MemoryView":704 * cdef assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim): * for suboffset in suboffsets[:ndim]: * if suboffset >= 0: # <<<<<<<<<<<<<< @@ -15903,7 +15966,7 @@ static PyObject *assert_direct_dimensions(Py_ssize_t *__pyx_v_suboffsets, int __ } } - /* "View.MemoryView":700 + /* "View.MemoryView":702 * return have_slices or nslices, tuple(result) * * cdef assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim): # <<<<<<<<<<<<<< @@ -15924,7 +15987,7 @@ static PyObject *assert_direct_dimensions(Py_ssize_t *__pyx_v_suboffsets, int __ return __pyx_r; } -/* "View.MemoryView":710 +/* "View.MemoryView":712 * * @cname('__pyx_memview_slice') * cdef memoryview memview_slice(memoryview memview, object indices): # <<<<<<<<<<<<<< @@ -15968,7 +16031,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("memview_slice", 0); - /* "View.MemoryView":711 + /* "View.MemoryView":713 * @cname('__pyx_memview_slice') * cdef memoryview memview_slice(memoryview memview, object indices): * cdef int new_ndim = 0, suboffset_dim = -1, dim # <<<<<<<<<<<<<< @@ -15978,7 +16041,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_v_new_ndim = 0; __pyx_v_suboffset_dim = -1; - /* "View.MemoryView":718 + /* "View.MemoryView":720 * * * memset(&dst, 0, sizeof(dst)) # <<<<<<<<<<<<<< @@ -15987,7 +16050,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ (void)(memset((&__pyx_v_dst), 0, (sizeof(__pyx_v_dst)))); - /* "View.MemoryView":722 + /* "View.MemoryView":724 * cdef _memoryviewslice memviewsliceobj * * assert memview.view.ndim > 0 # <<<<<<<<<<<<<< @@ -15998,12 +16061,12 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ if (unlikely(!Py_OptimizeFlag)) { if (unlikely(!((__pyx_v_memview->view.ndim > 0) != 0))) { PyErr_SetNone(PyExc_AssertionError); - __PYX_ERR(1, 722, __pyx_L1_error) + __PYX_ERR(1, 724, __pyx_L1_error) } } #endif - /* "View.MemoryView":724 + /* "View.MemoryView":726 * assert memview.view.ndim > 0 * * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< @@ -16014,20 +16077,20 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "View.MemoryView":725 + /* "View.MemoryView":727 * * if isinstance(memview, _memoryviewslice): * memviewsliceobj = memview # <<<<<<<<<<<<<< * p_src = &memviewsliceobj.from_slice * else: */ - if (!(likely(((((PyObject *)__pyx_v_memview)) == Py_None) || likely(__Pyx_TypeTest(((PyObject *)__pyx_v_memview), __pyx_memoryviewslice_type))))) __PYX_ERR(1, 725, __pyx_L1_error) + if (!(likely(((((PyObject *)__pyx_v_memview)) == Py_None) || likely(__Pyx_TypeTest(((PyObject *)__pyx_v_memview), __pyx_memoryviewslice_type))))) __PYX_ERR(1, 727, __pyx_L1_error) __pyx_t_3 = ((PyObject *)__pyx_v_memview); __Pyx_INCREF(__pyx_t_3); __pyx_v_memviewsliceobj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":726 + /* "View.MemoryView":728 * if isinstance(memview, _memoryviewslice): * memviewsliceobj = memview * p_src = &memviewsliceobj.from_slice # <<<<<<<<<<<<<< @@ -16036,7 +16099,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ __pyx_v_p_src = (&__pyx_v_memviewsliceobj->from_slice); - /* "View.MemoryView":724 + /* "View.MemoryView":726 * assert memview.view.ndim > 0 * * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< @@ -16046,7 +16109,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ goto __pyx_L3; } - /* "View.MemoryView":728 + /* "View.MemoryView":730 * p_src = &memviewsliceobj.from_slice * else: * slice_copy(memview, &src) # <<<<<<<<<<<<<< @@ -16056,7 +16119,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ /*else*/ { __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_src)); - /* "View.MemoryView":729 + /* "View.MemoryView":731 * else: * slice_copy(memview, &src) * p_src = &src # <<<<<<<<<<<<<< @@ -16067,7 +16130,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ } __pyx_L3:; - /* "View.MemoryView":735 + /* "View.MemoryView":737 * * * dst.memview = p_src.memview # <<<<<<<<<<<<<< @@ -16077,7 +16140,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_t_4 = __pyx_v_p_src->memview; __pyx_v_dst.memview = __pyx_t_4; - /* "View.MemoryView":736 + /* "View.MemoryView":738 * * dst.memview = p_src.memview * dst.data = p_src.data # <<<<<<<<<<<<<< @@ -16087,7 +16150,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_t_5 = __pyx_v_p_src->data; __pyx_v_dst.data = __pyx_t_5; - /* "View.MemoryView":741 + /* "View.MemoryView":743 * * * cdef __Pyx_memviewslice *p_dst = &dst # <<<<<<<<<<<<<< @@ -16096,7 +16159,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ __pyx_v_p_dst = (&__pyx_v_dst); - /* "View.MemoryView":742 + /* "View.MemoryView":744 * * cdef __Pyx_memviewslice *p_dst = &dst * cdef int *p_suboffset_dim = &suboffset_dim # <<<<<<<<<<<<<< @@ -16105,7 +16168,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ __pyx_v_p_suboffset_dim = (&__pyx_v_suboffset_dim); - /* "View.MemoryView":746 + /* "View.MemoryView":748 * cdef bint have_start, have_stop, have_step * * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< @@ -16117,26 +16180,26 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_t_3 = __pyx_v_indices; __Pyx_INCREF(__pyx_t_3); __pyx_t_7 = 0; __pyx_t_8 = NULL; } else { - __pyx_t_7 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_v_indices); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 746, __pyx_L1_error) + __pyx_t_7 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_v_indices); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 748, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_8 = Py_TYPE(__pyx_t_3)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 746, __pyx_L1_error) + __pyx_t_8 = Py_TYPE(__pyx_t_3)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 748, __pyx_L1_error) } for (;;) { if (likely(!__pyx_t_8)) { if (likely(PyList_CheckExact(__pyx_t_3))) { if (__pyx_t_7 >= PyList_GET_SIZE(__pyx_t_3)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_9 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_7); __Pyx_INCREF(__pyx_t_9); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(1, 746, __pyx_L1_error) + __pyx_t_9 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_7); __Pyx_INCREF(__pyx_t_9); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(1, 748, __pyx_L1_error) #else - __pyx_t_9 = PySequence_ITEM(__pyx_t_3, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 746, __pyx_L1_error) + __pyx_t_9 = PySequence_ITEM(__pyx_t_3, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 748, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); #endif } else { if (__pyx_t_7 >= PyTuple_GET_SIZE(__pyx_t_3)) break; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - __pyx_t_9 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_7); __Pyx_INCREF(__pyx_t_9); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(1, 746, __pyx_L1_error) + __pyx_t_9 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_7); __Pyx_INCREF(__pyx_t_9); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(1, 748, __pyx_L1_error) #else - __pyx_t_9 = PySequence_ITEM(__pyx_t_3, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 746, __pyx_L1_error) + __pyx_t_9 = PySequence_ITEM(__pyx_t_3, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 748, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); #endif } @@ -16146,7 +16209,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ PyObject* exc_type = PyErr_Occurred(); if (exc_type) { if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); - else __PYX_ERR(1, 746, __pyx_L1_error) + else __PYX_ERR(1, 748, __pyx_L1_error) } break; } @@ -16157,7 +16220,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_v_dim = __pyx_t_6; __pyx_t_6 = (__pyx_t_6 + 1); - /* "View.MemoryView":747 + /* "View.MemoryView":749 * * for dim, index in enumerate(indices): * if PyIndex_Check(index): # <<<<<<<<<<<<<< @@ -16167,25 +16230,25 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_t_2 = (PyIndex_Check(__pyx_v_index) != 0); if (__pyx_t_2) { - /* "View.MemoryView":751 + /* "View.MemoryView":753 * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], * dim, new_ndim, p_suboffset_dim, * index, 0, 0, # start, stop, step # <<<<<<<<<<<<<< * 0, 0, 0, # have_{start,stop,step} * False) */ - __pyx_t_10 = __Pyx_PyIndex_AsSsize_t(__pyx_v_index); if (unlikely((__pyx_t_10 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 751, __pyx_L1_error) + __pyx_t_10 = __Pyx_PyIndex_AsSsize_t(__pyx_v_index); if (unlikely((__pyx_t_10 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 753, __pyx_L1_error) - /* "View.MemoryView":748 + /* "View.MemoryView":750 * for dim, index in enumerate(indices): * if PyIndex_Check(index): * slice_memviewslice( # <<<<<<<<<<<<<< * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], * dim, new_ndim, p_suboffset_dim, */ - __pyx_t_11 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_t_10, 0, 0, 0, 0, 0, 0); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(1, 748, __pyx_L1_error) + __pyx_t_11 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_t_10, 0, 0, 0, 0, 0, 0); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(1, 750, __pyx_L1_error) - /* "View.MemoryView":747 + /* "View.MemoryView":749 * * for dim, index in enumerate(indices): * if PyIndex_Check(index): # <<<<<<<<<<<<<< @@ -16195,7 +16258,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ goto __pyx_L6; } - /* "View.MemoryView":754 + /* "View.MemoryView":756 * 0, 0, 0, # have_{start,stop,step} * False) * elif index is None: # <<<<<<<<<<<<<< @@ -16206,7 +16269,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_t_1 = (__pyx_t_2 != 0); if (__pyx_t_1) { - /* "View.MemoryView":755 + /* "View.MemoryView":757 * False) * elif index is None: * p_dst.shape[new_ndim] = 1 # <<<<<<<<<<<<<< @@ -16215,7 +16278,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ (__pyx_v_p_dst->shape[__pyx_v_new_ndim]) = 1; - /* "View.MemoryView":756 + /* "View.MemoryView":758 * elif index is None: * p_dst.shape[new_ndim] = 1 * p_dst.strides[new_ndim] = 0 # <<<<<<<<<<<<<< @@ -16224,7 +16287,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ (__pyx_v_p_dst->strides[__pyx_v_new_ndim]) = 0; - /* "View.MemoryView":757 + /* "View.MemoryView":759 * p_dst.shape[new_ndim] = 1 * p_dst.strides[new_ndim] = 0 * p_dst.suboffsets[new_ndim] = -1 # <<<<<<<<<<<<<< @@ -16233,7 +16296,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ (__pyx_v_p_dst->suboffsets[__pyx_v_new_ndim]) = -1L; - /* "View.MemoryView":758 + /* "View.MemoryView":760 * p_dst.strides[new_ndim] = 0 * p_dst.suboffsets[new_ndim] = -1 * new_ndim += 1 # <<<<<<<<<<<<<< @@ -16242,7 +16305,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ __pyx_v_new_ndim = (__pyx_v_new_ndim + 1); - /* "View.MemoryView":754 + /* "View.MemoryView":756 * 0, 0, 0, # have_{start,stop,step} * False) * elif index is None: # <<<<<<<<<<<<<< @@ -16252,7 +16315,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ goto __pyx_L6; } - /* "View.MemoryView":760 + /* "View.MemoryView":762 * new_ndim += 1 * else: * start = index.start or 0 # <<<<<<<<<<<<<< @@ -16260,13 +16323,13 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ * step = index.step or 0 */ /*else*/ { - __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_start); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 760, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_start); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 762, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_9); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 760, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_9); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 762, __pyx_L1_error) if (!__pyx_t_1) { __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } else { - __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_9); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 760, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_9); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 762, __pyx_L1_error) __pyx_t_10 = __pyx_t_12; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; goto __pyx_L7_bool_binop_done; @@ -16275,20 +16338,20 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_L7_bool_binop_done:; __pyx_v_start = __pyx_t_10; - /* "View.MemoryView":761 + /* "View.MemoryView":763 * else: * start = index.start or 0 * stop = index.stop or 0 # <<<<<<<<<<<<<< * step = index.step or 0 * */ - __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_stop); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 761, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_stop); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 763, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_9); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 761, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_9); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 763, __pyx_L1_error) if (!__pyx_t_1) { __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } else { - __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_9); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 761, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_9); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 763, __pyx_L1_error) __pyx_t_10 = __pyx_t_12; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; goto __pyx_L9_bool_binop_done; @@ -16297,20 +16360,20 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_L9_bool_binop_done:; __pyx_v_stop = __pyx_t_10; - /* "View.MemoryView":762 + /* "View.MemoryView":764 * start = index.start or 0 * stop = index.stop or 0 * step = index.step or 0 # <<<<<<<<<<<<<< * * have_start = index.start is not None */ - __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_step); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 762, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_step); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 764, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_9); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 762, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_9); if (unlikely(__pyx_t_1 < 0)) __PYX_ERR(1, 764, __pyx_L1_error) if (!__pyx_t_1) { __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; } else { - __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_9); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 762, __pyx_L1_error) + __pyx_t_12 = __Pyx_PyIndex_AsSsize_t(__pyx_t_9); if (unlikely((__pyx_t_12 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 764, __pyx_L1_error) __pyx_t_10 = __pyx_t_12; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; goto __pyx_L11_bool_binop_done; @@ -16319,55 +16382,55 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_L11_bool_binop_done:; __pyx_v_step = __pyx_t_10; - /* "View.MemoryView":764 + /* "View.MemoryView":766 * step = index.step or 0 * * have_start = index.start is not None # <<<<<<<<<<<<<< * have_stop = index.stop is not None * have_step = index.step is not None */ - __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_start); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 764, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_start); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 766, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); __pyx_t_1 = (__pyx_t_9 != Py_None); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __pyx_v_have_start = __pyx_t_1; - /* "View.MemoryView":765 + /* "View.MemoryView":767 * * have_start = index.start is not None * have_stop = index.stop is not None # <<<<<<<<<<<<<< * have_step = index.step is not None * */ - __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_stop); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 765, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_stop); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 767, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); __pyx_t_1 = (__pyx_t_9 != Py_None); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __pyx_v_have_stop = __pyx_t_1; - /* "View.MemoryView":766 + /* "View.MemoryView":768 * have_start = index.start is not None * have_stop = index.stop is not None * have_step = index.step is not None # <<<<<<<<<<<<<< * * slice_memviewslice( */ - __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_step); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 766, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_v_index, __pyx_n_s_step); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 768, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_9); __pyx_t_1 = (__pyx_t_9 != Py_None); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __pyx_v_have_step = __pyx_t_1; - /* "View.MemoryView":768 + /* "View.MemoryView":770 * have_step = index.step is not None * * slice_memviewslice( # <<<<<<<<<<<<<< * p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], * dim, new_ndim, p_suboffset_dim, */ - __pyx_t_11 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_start, __pyx_v_stop, __pyx_v_step, __pyx_v_have_start, __pyx_v_have_stop, __pyx_v_have_step, 1); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(1, 768, __pyx_L1_error) + __pyx_t_11 = __pyx_memoryview_slice_memviewslice(__pyx_v_p_dst, (__pyx_v_p_src->shape[__pyx_v_dim]), (__pyx_v_p_src->strides[__pyx_v_dim]), (__pyx_v_p_src->suboffsets[__pyx_v_dim]), __pyx_v_dim, __pyx_v_new_ndim, __pyx_v_p_suboffset_dim, __pyx_v_start, __pyx_v_stop, __pyx_v_step, __pyx_v_have_start, __pyx_v_have_stop, __pyx_v_have_step, 1); if (unlikely(__pyx_t_11 == ((int)-1))) __PYX_ERR(1, 770, __pyx_L1_error) - /* "View.MemoryView":774 + /* "View.MemoryView":776 * have_start, have_stop, have_step, * True) * new_ndim += 1 # <<<<<<<<<<<<<< @@ -16378,7 +16441,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ } __pyx_L6:; - /* "View.MemoryView":746 + /* "View.MemoryView":748 * cdef bint have_start, have_stop, have_step * * for dim, index in enumerate(indices): # <<<<<<<<<<<<<< @@ -16388,7 +16451,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ } __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":776 + /* "View.MemoryView":778 * new_ndim += 1 * * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< @@ -16399,7 +16462,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "View.MemoryView":777 + /* "View.MemoryView":779 * * if isinstance(memview, _memoryviewslice): * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< @@ -16408,39 +16471,39 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ __Pyx_XDECREF(((PyObject *)__pyx_r)); - /* "View.MemoryView":778 + /* "View.MemoryView":780 * if isinstance(memview, _memoryviewslice): * return memoryview_fromslice(dst, new_ndim, * memviewsliceobj.to_object_func, # <<<<<<<<<<<<<< * memviewsliceobj.to_dtype_func, * memview.dtype_is_object) */ - if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 778, __pyx_L1_error) } + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 780, __pyx_L1_error) } - /* "View.MemoryView":779 + /* "View.MemoryView":781 * return memoryview_fromslice(dst, new_ndim, * memviewsliceobj.to_object_func, * memviewsliceobj.to_dtype_func, # <<<<<<<<<<<<<< * memview.dtype_is_object) * else: */ - if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 779, __pyx_L1_error) } + if (unlikely(!__pyx_v_memviewsliceobj)) { __Pyx_RaiseUnboundLocalError("memviewsliceobj"); __PYX_ERR(1, 781, __pyx_L1_error) } - /* "View.MemoryView":777 + /* "View.MemoryView":779 * * if isinstance(memview, _memoryviewslice): * return memoryview_fromslice(dst, new_ndim, # <<<<<<<<<<<<<< * memviewsliceobj.to_object_func, * memviewsliceobj.to_dtype_func, */ - __pyx_t_3 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, __pyx_v_memviewsliceobj->to_object_func, __pyx_v_memviewsliceobj->to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 777, __pyx_L1_error) + __pyx_t_3 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, __pyx_v_memviewsliceobj->to_object_func, __pyx_v_memviewsliceobj->to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 779, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_memoryview_type))))) __PYX_ERR(1, 777, __pyx_L1_error) + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_memoryview_type))))) __PYX_ERR(1, 779, __pyx_L1_error) __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_3); __pyx_t_3 = 0; goto __pyx_L0; - /* "View.MemoryView":776 + /* "View.MemoryView":778 * new_ndim += 1 * * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< @@ -16449,7 +16512,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ */ } - /* "View.MemoryView":782 + /* "View.MemoryView":784 * memview.dtype_is_object) * else: * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< @@ -16459,30 +16522,30 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ /*else*/ { __Pyx_XDECREF(((PyObject *)__pyx_r)); - /* "View.MemoryView":783 + /* "View.MemoryView":785 * else: * return memoryview_fromslice(dst, new_ndim, NULL, NULL, * memview.dtype_is_object) # <<<<<<<<<<<<<< * * */ - __pyx_t_3 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, NULL, NULL, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 782, __pyx_L1_error) + __pyx_t_3 = __pyx_memoryview_fromslice(__pyx_v_dst, __pyx_v_new_ndim, NULL, NULL, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 784, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - /* "View.MemoryView":782 + /* "View.MemoryView":784 * memview.dtype_is_object) * else: * return memoryview_fromslice(dst, new_ndim, NULL, NULL, # <<<<<<<<<<<<<< * memview.dtype_is_object) * */ - if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_memoryview_type))))) __PYX_ERR(1, 782, __pyx_L1_error) + if (!(likely(((__pyx_t_3) == Py_None) || likely(__Pyx_TypeTest(__pyx_t_3, __pyx_memoryview_type))))) __PYX_ERR(1, 784, __pyx_L1_error) __pyx_r = ((struct __pyx_memoryview_obj *)__pyx_t_3); __pyx_t_3 = 0; goto __pyx_L0; } - /* "View.MemoryView":710 + /* "View.MemoryView":712 * * @cname('__pyx_memview_slice') * cdef memoryview memview_slice(memoryview memview, object indices): # <<<<<<<<<<<<<< @@ -16504,7 +16567,7 @@ static struct __pyx_memoryview_obj *__pyx_memview_slice(struct __pyx_memoryview_ return __pyx_r; } -/* "View.MemoryView":807 +/* "View.MemoryView":809 * * @cname('__pyx_memoryview_slice_memviewslice') * cdef int slice_memviewslice( # <<<<<<<<<<<<<< @@ -16523,7 +16586,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, const char *__pyx_filename = NULL; int __pyx_clineno = 0; - /* "View.MemoryView":827 + /* "View.MemoryView":829 * cdef bint negative_step * * if not is_slice: # <<<<<<<<<<<<<< @@ -16533,7 +16596,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_1 = ((!(__pyx_v_is_slice != 0)) != 0); if (__pyx_t_1) { - /* "View.MemoryView":829 + /* "View.MemoryView":831 * if not is_slice: * * if start < 0: # <<<<<<<<<<<<<< @@ -16543,7 +16606,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_1 = ((__pyx_v_start < 0) != 0); if (__pyx_t_1) { - /* "View.MemoryView":830 + /* "View.MemoryView":832 * * if start < 0: * start += shape # <<<<<<<<<<<<<< @@ -16552,7 +16615,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_start = (__pyx_v_start + __pyx_v_shape); - /* "View.MemoryView":829 + /* "View.MemoryView":831 * if not is_slice: * * if start < 0: # <<<<<<<<<<<<<< @@ -16561,7 +16624,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ } - /* "View.MemoryView":831 + /* "View.MemoryView":833 * if start < 0: * start += shape * if not 0 <= start < shape: # <<<<<<<<<<<<<< @@ -16575,16 +16638,16 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((!(__pyx_t_1 != 0)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":832 + /* "View.MemoryView":834 * start += shape * if not 0 <= start < shape: * _err_dim(IndexError, "Index out of bounds (axis %d)", dim) # <<<<<<<<<<<<<< * else: * */ - __pyx_t_3 = __pyx_memoryview_err_dim(__pyx_builtin_IndexError, ((char *)"Index out of bounds (axis %d)"), __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 832, __pyx_L1_error) + __pyx_t_3 = __pyx_memoryview_err_dim(__pyx_builtin_IndexError, ((char *)"Index out of bounds (axis %d)"), __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 834, __pyx_L1_error) - /* "View.MemoryView":831 + /* "View.MemoryView":833 * if start < 0: * start += shape * if not 0 <= start < shape: # <<<<<<<<<<<<<< @@ -16593,7 +16656,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ } - /* "View.MemoryView":827 + /* "View.MemoryView":829 * cdef bint negative_step * * if not is_slice: # <<<<<<<<<<<<<< @@ -16603,7 +16666,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L3; } - /* "View.MemoryView":835 + /* "View.MemoryView":837 * else: * * negative_step = have_step != 0 and step < 0 # <<<<<<<<<<<<<< @@ -16622,7 +16685,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_L6_bool_binop_done:; __pyx_v_negative_step = __pyx_t_2; - /* "View.MemoryView":837 + /* "View.MemoryView":839 * negative_step = have_step != 0 and step < 0 * * if have_step and step == 0: # <<<<<<<<<<<<<< @@ -16640,16 +16703,16 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_L9_bool_binop_done:; if (__pyx_t_2) { - /* "View.MemoryView":838 + /* "View.MemoryView":840 * * if have_step and step == 0: * _err_dim(ValueError, "Step may not be zero (axis %d)", dim) # <<<<<<<<<<<<<< * * */ - __pyx_t_3 = __pyx_memoryview_err_dim(__pyx_builtin_ValueError, ((char *)"Step may not be zero (axis %d)"), __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 838, __pyx_L1_error) + __pyx_t_3 = __pyx_memoryview_err_dim(__pyx_builtin_ValueError, ((char *)"Step may not be zero (axis %d)"), __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 840, __pyx_L1_error) - /* "View.MemoryView":837 + /* "View.MemoryView":839 * negative_step = have_step != 0 and step < 0 * * if have_step and step == 0: # <<<<<<<<<<<<<< @@ -16658,7 +16721,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ } - /* "View.MemoryView":841 + /* "View.MemoryView":843 * * * if have_start: # <<<<<<<<<<<<<< @@ -16668,7 +16731,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = (__pyx_v_have_start != 0); if (__pyx_t_2) { - /* "View.MemoryView":842 + /* "View.MemoryView":844 * * if have_start: * if start < 0: # <<<<<<<<<<<<<< @@ -16678,7 +16741,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((__pyx_v_start < 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":843 + /* "View.MemoryView":845 * if have_start: * if start < 0: * start += shape # <<<<<<<<<<<<<< @@ -16687,7 +16750,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_start = (__pyx_v_start + __pyx_v_shape); - /* "View.MemoryView":844 + /* "View.MemoryView":846 * if start < 0: * start += shape * if start < 0: # <<<<<<<<<<<<<< @@ -16697,7 +16760,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((__pyx_v_start < 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":845 + /* "View.MemoryView":847 * start += shape * if start < 0: * start = 0 # <<<<<<<<<<<<<< @@ -16706,7 +16769,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_start = 0; - /* "View.MemoryView":844 + /* "View.MemoryView":846 * if start < 0: * start += shape * if start < 0: # <<<<<<<<<<<<<< @@ -16715,7 +16778,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ } - /* "View.MemoryView":842 + /* "View.MemoryView":844 * * if have_start: * if start < 0: # <<<<<<<<<<<<<< @@ -16725,7 +16788,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L12; } - /* "View.MemoryView":846 + /* "View.MemoryView":848 * if start < 0: * start = 0 * elif start >= shape: # <<<<<<<<<<<<<< @@ -16735,7 +16798,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((__pyx_v_start >= __pyx_v_shape) != 0); if (__pyx_t_2) { - /* "View.MemoryView":847 + /* "View.MemoryView":849 * start = 0 * elif start >= shape: * if negative_step: # <<<<<<<<<<<<<< @@ -16745,7 +16808,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = (__pyx_v_negative_step != 0); if (__pyx_t_2) { - /* "View.MemoryView":848 + /* "View.MemoryView":850 * elif start >= shape: * if negative_step: * start = shape - 1 # <<<<<<<<<<<<<< @@ -16754,7 +16817,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_start = (__pyx_v_shape - 1); - /* "View.MemoryView":847 + /* "View.MemoryView":849 * start = 0 * elif start >= shape: * if negative_step: # <<<<<<<<<<<<<< @@ -16764,7 +16827,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L14; } - /* "View.MemoryView":850 + /* "View.MemoryView":852 * start = shape - 1 * else: * start = shape # <<<<<<<<<<<<<< @@ -16776,7 +16839,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, } __pyx_L14:; - /* "View.MemoryView":846 + /* "View.MemoryView":848 * if start < 0: * start = 0 * elif start >= shape: # <<<<<<<<<<<<<< @@ -16786,7 +16849,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, } __pyx_L12:; - /* "View.MemoryView":841 + /* "View.MemoryView":843 * * * if have_start: # <<<<<<<<<<<<<< @@ -16796,7 +16859,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L11; } - /* "View.MemoryView":852 + /* "View.MemoryView":854 * start = shape * else: * if negative_step: # <<<<<<<<<<<<<< @@ -16807,7 +16870,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = (__pyx_v_negative_step != 0); if (__pyx_t_2) { - /* "View.MemoryView":853 + /* "View.MemoryView":855 * else: * if negative_step: * start = shape - 1 # <<<<<<<<<<<<<< @@ -16816,7 +16879,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_start = (__pyx_v_shape - 1); - /* "View.MemoryView":852 + /* "View.MemoryView":854 * start = shape * else: * if negative_step: # <<<<<<<<<<<<<< @@ -16826,7 +16889,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L15; } - /* "View.MemoryView":855 + /* "View.MemoryView":857 * start = shape - 1 * else: * start = 0 # <<<<<<<<<<<<<< @@ -16840,7 +16903,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, } __pyx_L11:; - /* "View.MemoryView":857 + /* "View.MemoryView":859 * start = 0 * * if have_stop: # <<<<<<<<<<<<<< @@ -16850,7 +16913,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = (__pyx_v_have_stop != 0); if (__pyx_t_2) { - /* "View.MemoryView":858 + /* "View.MemoryView":860 * * if have_stop: * if stop < 0: # <<<<<<<<<<<<<< @@ -16860,7 +16923,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((__pyx_v_stop < 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":859 + /* "View.MemoryView":861 * if have_stop: * if stop < 0: * stop += shape # <<<<<<<<<<<<<< @@ -16869,7 +16932,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_stop = (__pyx_v_stop + __pyx_v_shape); - /* "View.MemoryView":860 + /* "View.MemoryView":862 * if stop < 0: * stop += shape * if stop < 0: # <<<<<<<<<<<<<< @@ -16879,7 +16942,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((__pyx_v_stop < 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":861 + /* "View.MemoryView":863 * stop += shape * if stop < 0: * stop = 0 # <<<<<<<<<<<<<< @@ -16888,7 +16951,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_stop = 0; - /* "View.MemoryView":860 + /* "View.MemoryView":862 * if stop < 0: * stop += shape * if stop < 0: # <<<<<<<<<<<<<< @@ -16897,7 +16960,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ } - /* "View.MemoryView":858 + /* "View.MemoryView":860 * * if have_stop: * if stop < 0: # <<<<<<<<<<<<<< @@ -16907,7 +16970,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L17; } - /* "View.MemoryView":862 + /* "View.MemoryView":864 * if stop < 0: * stop = 0 * elif stop > shape: # <<<<<<<<<<<<<< @@ -16917,7 +16980,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((__pyx_v_stop > __pyx_v_shape) != 0); if (__pyx_t_2) { - /* "View.MemoryView":863 + /* "View.MemoryView":865 * stop = 0 * elif stop > shape: * stop = shape # <<<<<<<<<<<<<< @@ -16926,7 +16989,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_stop = __pyx_v_shape; - /* "View.MemoryView":862 + /* "View.MemoryView":864 * if stop < 0: * stop = 0 * elif stop > shape: # <<<<<<<<<<<<<< @@ -16936,7 +16999,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, } __pyx_L17:; - /* "View.MemoryView":857 + /* "View.MemoryView":859 * start = 0 * * if have_stop: # <<<<<<<<<<<<<< @@ -16946,7 +17009,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L16; } - /* "View.MemoryView":865 + /* "View.MemoryView":867 * stop = shape * else: * if negative_step: # <<<<<<<<<<<<<< @@ -16957,7 +17020,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = (__pyx_v_negative_step != 0); if (__pyx_t_2) { - /* "View.MemoryView":866 + /* "View.MemoryView":868 * else: * if negative_step: * stop = -1 # <<<<<<<<<<<<<< @@ -16966,7 +17029,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_stop = -1L; - /* "View.MemoryView":865 + /* "View.MemoryView":867 * stop = shape * else: * if negative_step: # <<<<<<<<<<<<<< @@ -16976,7 +17039,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L19; } - /* "View.MemoryView":868 + /* "View.MemoryView":870 * stop = -1 * else: * stop = shape # <<<<<<<<<<<<<< @@ -16990,7 +17053,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, } __pyx_L16:; - /* "View.MemoryView":870 + /* "View.MemoryView":872 * stop = shape * * if not have_step: # <<<<<<<<<<<<<< @@ -17000,7 +17063,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((!(__pyx_v_have_step != 0)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":871 + /* "View.MemoryView":873 * * if not have_step: * step = 1 # <<<<<<<<<<<<<< @@ -17009,7 +17072,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_step = 1; - /* "View.MemoryView":870 + /* "View.MemoryView":872 * stop = shape * * if not have_step: # <<<<<<<<<<<<<< @@ -17018,7 +17081,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ } - /* "View.MemoryView":875 + /* "View.MemoryView":877 * * with cython.cdivision(True): * new_shape = (stop - start) // step # <<<<<<<<<<<<<< @@ -17027,7 +17090,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_new_shape = ((__pyx_v_stop - __pyx_v_start) / __pyx_v_step); - /* "View.MemoryView":877 + /* "View.MemoryView":879 * new_shape = (stop - start) // step * * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< @@ -17037,7 +17100,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = (((__pyx_v_stop - __pyx_v_start) - (__pyx_v_step * __pyx_v_new_shape)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":878 + /* "View.MemoryView":880 * * if (stop - start) - step * new_shape: * new_shape += 1 # <<<<<<<<<<<<<< @@ -17046,7 +17109,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_new_shape = (__pyx_v_new_shape + 1); - /* "View.MemoryView":877 + /* "View.MemoryView":879 * new_shape = (stop - start) // step * * if (stop - start) - step * new_shape: # <<<<<<<<<<<<<< @@ -17055,7 +17118,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ } - /* "View.MemoryView":880 + /* "View.MemoryView":882 * new_shape += 1 * * if new_shape < 0: # <<<<<<<<<<<<<< @@ -17065,7 +17128,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((__pyx_v_new_shape < 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":881 + /* "View.MemoryView":883 * * if new_shape < 0: * new_shape = 0 # <<<<<<<<<<<<<< @@ -17074,7 +17137,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_new_shape = 0; - /* "View.MemoryView":880 + /* "View.MemoryView":882 * new_shape += 1 * * if new_shape < 0: # <<<<<<<<<<<<<< @@ -17083,7 +17146,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ } - /* "View.MemoryView":884 + /* "View.MemoryView":886 * * * dst.strides[new_ndim] = stride * step # <<<<<<<<<<<<<< @@ -17092,7 +17155,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ (__pyx_v_dst->strides[__pyx_v_new_ndim]) = (__pyx_v_stride * __pyx_v_step); - /* "View.MemoryView":885 + /* "View.MemoryView":887 * * dst.strides[new_ndim] = stride * step * dst.shape[new_ndim] = new_shape # <<<<<<<<<<<<<< @@ -17101,7 +17164,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ (__pyx_v_dst->shape[__pyx_v_new_ndim]) = __pyx_v_new_shape; - /* "View.MemoryView":886 + /* "View.MemoryView":888 * dst.strides[new_ndim] = stride * step * dst.shape[new_ndim] = new_shape * dst.suboffsets[new_ndim] = suboffset # <<<<<<<<<<<<<< @@ -17112,7 +17175,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, } __pyx_L3:; - /* "View.MemoryView":889 + /* "View.MemoryView":891 * * * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< @@ -17122,7 +17185,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = (((__pyx_v_suboffset_dim[0]) < 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":890 + /* "View.MemoryView":892 * * if suboffset_dim[0] < 0: * dst.data += start * stride # <<<<<<<<<<<<<< @@ -17131,7 +17194,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_dst->data = (__pyx_v_dst->data + (__pyx_v_start * __pyx_v_stride)); - /* "View.MemoryView":889 + /* "View.MemoryView":891 * * * if suboffset_dim[0] < 0: # <<<<<<<<<<<<<< @@ -17141,7 +17204,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L23; } - /* "View.MemoryView":892 + /* "View.MemoryView":894 * dst.data += start * stride * else: * dst.suboffsets[suboffset_dim[0]] += start * stride # <<<<<<<<<<<<<< @@ -17154,7 +17217,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, } __pyx_L23:; - /* "View.MemoryView":894 + /* "View.MemoryView":896 * dst.suboffsets[suboffset_dim[0]] += start * stride * * if suboffset >= 0: # <<<<<<<<<<<<<< @@ -17164,7 +17227,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((__pyx_v_suboffset >= 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":895 + /* "View.MemoryView":897 * * if suboffset >= 0: * if not is_slice: # <<<<<<<<<<<<<< @@ -17174,7 +17237,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((!(__pyx_v_is_slice != 0)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":896 + /* "View.MemoryView":898 * if suboffset >= 0: * if not is_slice: * if new_ndim == 0: # <<<<<<<<<<<<<< @@ -17184,7 +17247,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_t_2 = ((__pyx_v_new_ndim == 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":897 + /* "View.MemoryView":899 * if not is_slice: * if new_ndim == 0: * dst.data = ( dst.data)[0] + suboffset # <<<<<<<<<<<<<< @@ -17193,7 +17256,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ __pyx_v_dst->data = ((((char **)__pyx_v_dst->data)[0]) + __pyx_v_suboffset); - /* "View.MemoryView":896 + /* "View.MemoryView":898 * if suboffset >= 0: * if not is_slice: * if new_ndim == 0: # <<<<<<<<<<<<<< @@ -17203,7 +17266,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L26; } - /* "View.MemoryView":899 + /* "View.MemoryView":901 * dst.data = ( dst.data)[0] + suboffset * else: * _err_dim(IndexError, "All dimensions preceding dimension %d " # <<<<<<<<<<<<<< @@ -17212,18 +17275,18 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ /*else*/ { - /* "View.MemoryView":900 + /* "View.MemoryView":902 * else: * _err_dim(IndexError, "All dimensions preceding dimension %d " * "must be indexed and not sliced", dim) # <<<<<<<<<<<<<< * else: * suboffset_dim[0] = new_ndim */ - __pyx_t_3 = __pyx_memoryview_err_dim(__pyx_builtin_IndexError, ((char *)"All dimensions preceding dimension %d must be indexed and not sliced"), __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 899, __pyx_L1_error) + __pyx_t_3 = __pyx_memoryview_err_dim(__pyx_builtin_IndexError, ((char *)"All dimensions preceding dimension %d must be indexed and not sliced"), __pyx_v_dim); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 901, __pyx_L1_error) } __pyx_L26:; - /* "View.MemoryView":895 + /* "View.MemoryView":897 * * if suboffset >= 0: * if not is_slice: # <<<<<<<<<<<<<< @@ -17233,7 +17296,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, goto __pyx_L25; } - /* "View.MemoryView":902 + /* "View.MemoryView":904 * "must be indexed and not sliced", dim) * else: * suboffset_dim[0] = new_ndim # <<<<<<<<<<<<<< @@ -17245,7 +17308,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, } __pyx_L25:; - /* "View.MemoryView":894 + /* "View.MemoryView":896 * dst.suboffsets[suboffset_dim[0]] += start * stride * * if suboffset >= 0: # <<<<<<<<<<<<<< @@ -17254,7 +17317,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, */ } - /* "View.MemoryView":904 + /* "View.MemoryView":906 * suboffset_dim[0] = new_ndim * * return 0 # <<<<<<<<<<<<<< @@ -17264,7 +17327,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, __pyx_r = 0; goto __pyx_L0; - /* "View.MemoryView":807 + /* "View.MemoryView":809 * * @cname('__pyx_memoryview_slice_memviewslice') * cdef int slice_memviewslice( # <<<<<<<<<<<<<< @@ -17288,7 +17351,7 @@ static int __pyx_memoryview_slice_memviewslice(__Pyx_memviewslice *__pyx_v_dst, return __pyx_r; } -/* "View.MemoryView":910 +/* "View.MemoryView":912 * * @cname('__pyx_pybuffer_index') * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, # <<<<<<<<<<<<<< @@ -17313,7 +17376,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P int __pyx_clineno = 0; __Pyx_RefNannySetupContext("pybuffer_index", 0); - /* "View.MemoryView":912 + /* "View.MemoryView":914 * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, * Py_ssize_t dim) except NULL: * cdef Py_ssize_t shape, stride, suboffset = -1 # <<<<<<<<<<<<<< @@ -17322,7 +17385,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ __pyx_v_suboffset = -1L; - /* "View.MemoryView":913 + /* "View.MemoryView":915 * Py_ssize_t dim) except NULL: * cdef Py_ssize_t shape, stride, suboffset = -1 * cdef Py_ssize_t itemsize = view.itemsize # <<<<<<<<<<<<<< @@ -17332,7 +17395,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P __pyx_t_1 = __pyx_v_view->itemsize; __pyx_v_itemsize = __pyx_t_1; - /* "View.MemoryView":916 + /* "View.MemoryView":918 * cdef char *resultp * * if view.ndim == 0: # <<<<<<<<<<<<<< @@ -17342,7 +17405,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P __pyx_t_2 = ((__pyx_v_view->ndim == 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":917 + /* "View.MemoryView":919 * * if view.ndim == 0: * shape = view.len / itemsize # <<<<<<<<<<<<<< @@ -17351,15 +17414,15 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ if (unlikely(__pyx_v_itemsize == 0)) { PyErr_SetString(PyExc_ZeroDivisionError, "integer division or modulo by zero"); - __PYX_ERR(1, 917, __pyx_L1_error) + __PYX_ERR(1, 919, __pyx_L1_error) } else if (sizeof(Py_ssize_t) == sizeof(long) && (!(((Py_ssize_t)-1) > 0)) && unlikely(__pyx_v_itemsize == (Py_ssize_t)-1) && unlikely(UNARY_NEG_WOULD_OVERFLOW(__pyx_v_view->len))) { PyErr_SetString(PyExc_OverflowError, "value too large to perform division"); - __PYX_ERR(1, 917, __pyx_L1_error) + __PYX_ERR(1, 919, __pyx_L1_error) } __pyx_v_shape = __Pyx_div_Py_ssize_t(__pyx_v_view->len, __pyx_v_itemsize); - /* "View.MemoryView":918 + /* "View.MemoryView":920 * if view.ndim == 0: * shape = view.len / itemsize * stride = itemsize # <<<<<<<<<<<<<< @@ -17368,7 +17431,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ __pyx_v_stride = __pyx_v_itemsize; - /* "View.MemoryView":916 + /* "View.MemoryView":918 * cdef char *resultp * * if view.ndim == 0: # <<<<<<<<<<<<<< @@ -17378,7 +17441,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P goto __pyx_L3; } - /* "View.MemoryView":920 + /* "View.MemoryView":922 * stride = itemsize * else: * shape = view.shape[dim] # <<<<<<<<<<<<<< @@ -17388,7 +17451,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P /*else*/ { __pyx_v_shape = (__pyx_v_view->shape[__pyx_v_dim]); - /* "View.MemoryView":921 + /* "View.MemoryView":923 * else: * shape = view.shape[dim] * stride = view.strides[dim] # <<<<<<<<<<<<<< @@ -17397,7 +17460,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ __pyx_v_stride = (__pyx_v_view->strides[__pyx_v_dim]); - /* "View.MemoryView":922 + /* "View.MemoryView":924 * shape = view.shape[dim] * stride = view.strides[dim] * if view.suboffsets != NULL: # <<<<<<<<<<<<<< @@ -17407,7 +17470,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P __pyx_t_2 = ((__pyx_v_view->suboffsets != NULL) != 0); if (__pyx_t_2) { - /* "View.MemoryView":923 + /* "View.MemoryView":925 * stride = view.strides[dim] * if view.suboffsets != NULL: * suboffset = view.suboffsets[dim] # <<<<<<<<<<<<<< @@ -17416,7 +17479,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ __pyx_v_suboffset = (__pyx_v_view->suboffsets[__pyx_v_dim]); - /* "View.MemoryView":922 + /* "View.MemoryView":924 * shape = view.shape[dim] * stride = view.strides[dim] * if view.suboffsets != NULL: # <<<<<<<<<<<<<< @@ -17427,7 +17490,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P } __pyx_L3:; - /* "View.MemoryView":925 + /* "View.MemoryView":927 * suboffset = view.suboffsets[dim] * * if index < 0: # <<<<<<<<<<<<<< @@ -17437,7 +17500,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P __pyx_t_2 = ((__pyx_v_index < 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":926 + /* "View.MemoryView":928 * * if index < 0: * index += view.shape[dim] # <<<<<<<<<<<<<< @@ -17446,7 +17509,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ __pyx_v_index = (__pyx_v_index + (__pyx_v_view->shape[__pyx_v_dim])); - /* "View.MemoryView":927 + /* "View.MemoryView":929 * if index < 0: * index += view.shape[dim] * if index < 0: # <<<<<<<<<<<<<< @@ -17456,26 +17519,26 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P __pyx_t_2 = ((__pyx_v_index < 0) != 0); if (unlikely(__pyx_t_2)) { - /* "View.MemoryView":928 + /* "View.MemoryView":930 * index += view.shape[dim] * if index < 0: * raise IndexError("Out of bounds on buffer access (axis %d)" % dim) # <<<<<<<<<<<<<< * * if index >= shape: */ - __pyx_t_3 = PyInt_FromSsize_t(__pyx_v_dim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 928, __pyx_L1_error) + __pyx_t_3 = PyInt_FromSsize_t(__pyx_v_dim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 930, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Out_of_bounds_on_buffer_access_a, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 928, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Out_of_bounds_on_buffer_access_a, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 930, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_builtin_IndexError, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 928, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_builtin_IndexError, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 930, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 928, __pyx_L1_error) + __PYX_ERR(1, 930, __pyx_L1_error) - /* "View.MemoryView":927 + /* "View.MemoryView":929 * if index < 0: * index += view.shape[dim] * if index < 0: # <<<<<<<<<<<<<< @@ -17484,7 +17547,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ } - /* "View.MemoryView":925 + /* "View.MemoryView":927 * suboffset = view.suboffsets[dim] * * if index < 0: # <<<<<<<<<<<<<< @@ -17493,7 +17556,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ } - /* "View.MemoryView":930 + /* "View.MemoryView":932 * raise IndexError("Out of bounds on buffer access (axis %d)" % dim) * * if index >= shape: # <<<<<<<<<<<<<< @@ -17503,26 +17566,26 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P __pyx_t_2 = ((__pyx_v_index >= __pyx_v_shape) != 0); if (unlikely(__pyx_t_2)) { - /* "View.MemoryView":931 + /* "View.MemoryView":933 * * if index >= shape: * raise IndexError("Out of bounds on buffer access (axis %d)" % dim) # <<<<<<<<<<<<<< * * resultp = bufp + index * stride */ - __pyx_t_3 = PyInt_FromSsize_t(__pyx_v_dim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 931, __pyx_L1_error) + __pyx_t_3 = PyInt_FromSsize_t(__pyx_v_dim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 933, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Out_of_bounds_on_buffer_access_a, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 931, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Out_of_bounds_on_buffer_access_a, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 933, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_builtin_IndexError, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 931, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_builtin_IndexError, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 933, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_Raise(__pyx_t_3, 0, 0, 0); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __PYX_ERR(1, 931, __pyx_L1_error) + __PYX_ERR(1, 933, __pyx_L1_error) - /* "View.MemoryView":930 + /* "View.MemoryView":932 * raise IndexError("Out of bounds on buffer access (axis %d)" % dim) * * if index >= shape: # <<<<<<<<<<<<<< @@ -17531,7 +17594,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ } - /* "View.MemoryView":933 + /* "View.MemoryView":935 * raise IndexError("Out of bounds on buffer access (axis %d)" % dim) * * resultp = bufp + index * stride # <<<<<<<<<<<<<< @@ -17540,7 +17603,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ __pyx_v_resultp = (__pyx_v_bufp + (__pyx_v_index * __pyx_v_stride)); - /* "View.MemoryView":934 + /* "View.MemoryView":936 * * resultp = bufp + index * stride * if suboffset >= 0: # <<<<<<<<<<<<<< @@ -17550,7 +17613,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P __pyx_t_2 = ((__pyx_v_suboffset >= 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":935 + /* "View.MemoryView":937 * resultp = bufp + index * stride * if suboffset >= 0: * resultp = ( resultp)[0] + suboffset # <<<<<<<<<<<<<< @@ -17559,7 +17622,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ __pyx_v_resultp = ((((char **)__pyx_v_resultp)[0]) + __pyx_v_suboffset); - /* "View.MemoryView":934 + /* "View.MemoryView":936 * * resultp = bufp + index * stride * if suboffset >= 0: # <<<<<<<<<<<<<< @@ -17568,7 +17631,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P */ } - /* "View.MemoryView":937 + /* "View.MemoryView":939 * resultp = ( resultp)[0] + suboffset * * return resultp # <<<<<<<<<<<<<< @@ -17578,7 +17641,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P __pyx_r = __pyx_v_resultp; goto __pyx_L0; - /* "View.MemoryView":910 + /* "View.MemoryView":912 * * @cname('__pyx_pybuffer_index') * cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, # <<<<<<<<<<<<<< @@ -17597,7 +17660,7 @@ static char *__pyx_pybuffer_index(Py_buffer *__pyx_v_view, char *__pyx_v_bufp, P return __pyx_r; } -/* "View.MemoryView":943 +/* "View.MemoryView":945 * * @cname('__pyx_memslice_transpose') * cdef int transpose_memslice(__Pyx_memviewslice *memslice) nogil except 0: # <<<<<<<<<<<<<< @@ -17625,7 +17688,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { const char *__pyx_filename = NULL; int __pyx_clineno = 0; - /* "View.MemoryView":944 + /* "View.MemoryView":946 * @cname('__pyx_memslice_transpose') * cdef int transpose_memslice(__Pyx_memviewslice *memslice) nogil except 0: * cdef int ndim = memslice.memview.view.ndim # <<<<<<<<<<<<<< @@ -17635,7 +17698,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { __pyx_t_1 = __pyx_v_memslice->memview->view.ndim; __pyx_v_ndim = __pyx_t_1; - /* "View.MemoryView":946 + /* "View.MemoryView":948 * cdef int ndim = memslice.memview.view.ndim * * cdef Py_ssize_t *shape = memslice.shape # <<<<<<<<<<<<<< @@ -17645,7 +17708,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { __pyx_t_2 = __pyx_v_memslice->shape; __pyx_v_shape = __pyx_t_2; - /* "View.MemoryView":947 + /* "View.MemoryView":949 * * cdef Py_ssize_t *shape = memslice.shape * cdef Py_ssize_t *strides = memslice.strides # <<<<<<<<<<<<<< @@ -17655,7 +17718,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { __pyx_t_2 = __pyx_v_memslice->strides; __pyx_v_strides = __pyx_t_2; - /* "View.MemoryView":951 + /* "View.MemoryView":953 * * cdef int i, j * for i in range(ndim / 2): # <<<<<<<<<<<<<< @@ -17667,7 +17730,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_4; __pyx_t_1+=1) { __pyx_v_i = __pyx_t_1; - /* "View.MemoryView":952 + /* "View.MemoryView":954 * cdef int i, j * for i in range(ndim / 2): * j = ndim - 1 - i # <<<<<<<<<<<<<< @@ -17676,7 +17739,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { */ __pyx_v_j = ((__pyx_v_ndim - 1) - __pyx_v_i); - /* "View.MemoryView":953 + /* "View.MemoryView":955 * for i in range(ndim / 2): * j = ndim - 1 - i * strides[i], strides[j] = strides[j], strides[i] # <<<<<<<<<<<<<< @@ -17688,7 +17751,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { (__pyx_v_strides[__pyx_v_i]) = __pyx_t_5; (__pyx_v_strides[__pyx_v_j]) = __pyx_t_6; - /* "View.MemoryView":954 + /* "View.MemoryView":956 * j = ndim - 1 - i * strides[i], strides[j] = strides[j], strides[i] * shape[i], shape[j] = shape[j], shape[i] # <<<<<<<<<<<<<< @@ -17700,7 +17763,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { (__pyx_v_shape[__pyx_v_i]) = __pyx_t_6; (__pyx_v_shape[__pyx_v_j]) = __pyx_t_5; - /* "View.MemoryView":956 + /* "View.MemoryView":958 * shape[i], shape[j] = shape[j], shape[i] * * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< @@ -17718,16 +17781,16 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { __pyx_L6_bool_binop_done:; if (__pyx_t_7) { - /* "View.MemoryView":957 + /* "View.MemoryView":959 * * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: * _err(ValueError, "Cannot transpose memoryview with indirect dimensions") # <<<<<<<<<<<<<< * * return 1 */ - __pyx_t_9 = __pyx_memoryview_err(__pyx_builtin_ValueError, ((char *)"Cannot transpose memoryview with indirect dimensions")); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 957, __pyx_L1_error) + __pyx_t_9 = __pyx_memoryview_err(__pyx_builtin_ValueError, ((char *)"Cannot transpose memoryview with indirect dimensions")); if (unlikely(__pyx_t_9 == ((int)-1))) __PYX_ERR(1, 959, __pyx_L1_error) - /* "View.MemoryView":956 + /* "View.MemoryView":958 * shape[i], shape[j] = shape[j], shape[i] * * if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: # <<<<<<<<<<<<<< @@ -17737,7 +17800,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { } } - /* "View.MemoryView":959 + /* "View.MemoryView":961 * _err(ValueError, "Cannot transpose memoryview with indirect dimensions") * * return 1 # <<<<<<<<<<<<<< @@ -17747,7 +17810,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { __pyx_r = 1; goto __pyx_L0; - /* "View.MemoryView":943 + /* "View.MemoryView":945 * * @cname('__pyx_memslice_transpose') * cdef int transpose_memslice(__Pyx_memviewslice *memslice) nogil except 0: # <<<<<<<<<<<<<< @@ -17771,7 +17834,7 @@ static int __pyx_memslice_transpose(__Pyx_memviewslice *__pyx_v_memslice) { return __pyx_r; } -/* "View.MemoryView":976 +/* "View.MemoryView":978 * cdef int (*to_dtype_func)(char *, object) except 0 * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -17794,7 +17857,7 @@ static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewsl __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "View.MemoryView":977 + /* "View.MemoryView":979 * * def __dealloc__(self): * __PYX_XDEC_MEMVIEW(&self.from_slice, 1) # <<<<<<<<<<<<<< @@ -17803,7 +17866,7 @@ static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewsl */ __PYX_XDEC_MEMVIEW((&__pyx_v_self->from_slice), 1); - /* "View.MemoryView":976 + /* "View.MemoryView":978 * cdef int (*to_dtype_func)(char *, object) except 0 * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -17815,7 +17878,7 @@ static void __pyx_memoryviewslice___pyx_pf_15View_dot_MemoryView_16_memoryviewsl __Pyx_RefNannyFinishContext(); } -/* "View.MemoryView":979 +/* "View.MemoryView":981 * __PYX_XDEC_MEMVIEW(&self.from_slice, 1) * * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< @@ -17833,7 +17896,7 @@ static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memor int __pyx_clineno = 0; __Pyx_RefNannySetupContext("convert_item_to_object", 0); - /* "View.MemoryView":980 + /* "View.MemoryView":982 * * cdef convert_item_to_object(self, char *itemp): * if self.to_object_func != NULL: # <<<<<<<<<<<<<< @@ -17843,7 +17906,7 @@ static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memor __pyx_t_1 = ((__pyx_v_self->to_object_func != NULL) != 0); if (__pyx_t_1) { - /* "View.MemoryView":981 + /* "View.MemoryView":983 * cdef convert_item_to_object(self, char *itemp): * if self.to_object_func != NULL: * return self.to_object_func(itemp) # <<<<<<<<<<<<<< @@ -17851,13 +17914,13 @@ static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memor * return memoryview.convert_item_to_object(self, itemp) */ __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_v_self->to_object_func(__pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 981, __pyx_L1_error) + __pyx_t_2 = __pyx_v_self->to_object_func(__pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 983, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "View.MemoryView":980 + /* "View.MemoryView":982 * * cdef convert_item_to_object(self, char *itemp): * if self.to_object_func != NULL: # <<<<<<<<<<<<<< @@ -17866,7 +17929,7 @@ static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memor */ } - /* "View.MemoryView":983 + /* "View.MemoryView":985 * return self.to_object_func(itemp) * else: * return memoryview.convert_item_to_object(self, itemp) # <<<<<<<<<<<<<< @@ -17875,14 +17938,14 @@ static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memor */ /*else*/ { __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_memoryview_convert_item_to_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 983, __pyx_L1_error) + __pyx_t_2 = __pyx_memoryview_convert_item_to_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 985, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; } - /* "View.MemoryView":979 + /* "View.MemoryView":981 * __PYX_XDEC_MEMVIEW(&self.from_slice, 1) * * cdef convert_item_to_object(self, char *itemp): # <<<<<<<<<<<<<< @@ -17901,7 +17964,7 @@ static PyObject *__pyx_memoryviewslice_convert_item_to_object(struct __pyx_memor return __pyx_r; } -/* "View.MemoryView":985 +/* "View.MemoryView":987 * return memoryview.convert_item_to_object(self, itemp) * * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< @@ -17920,7 +17983,7 @@ static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memo int __pyx_clineno = 0; __Pyx_RefNannySetupContext("assign_item_from_object", 0); - /* "View.MemoryView":986 + /* "View.MemoryView":988 * * cdef assign_item_from_object(self, char *itemp, object value): * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< @@ -17930,16 +17993,16 @@ static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memo __pyx_t_1 = ((__pyx_v_self->to_dtype_func != NULL) != 0); if (__pyx_t_1) { - /* "View.MemoryView":987 + /* "View.MemoryView":989 * cdef assign_item_from_object(self, char *itemp, object value): * if self.to_dtype_func != NULL: * self.to_dtype_func(itemp, value) # <<<<<<<<<<<<<< * else: * memoryview.assign_item_from_object(self, itemp, value) */ - __pyx_t_2 = __pyx_v_self->to_dtype_func(__pyx_v_itemp, __pyx_v_value); if (unlikely(__pyx_t_2 == ((int)0))) __PYX_ERR(1, 987, __pyx_L1_error) + __pyx_t_2 = __pyx_v_self->to_dtype_func(__pyx_v_itemp, __pyx_v_value); if (unlikely(__pyx_t_2 == ((int)0))) __PYX_ERR(1, 989, __pyx_L1_error) - /* "View.MemoryView":986 + /* "View.MemoryView":988 * * cdef assign_item_from_object(self, char *itemp, object value): * if self.to_dtype_func != NULL: # <<<<<<<<<<<<<< @@ -17949,7 +18012,7 @@ static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memo goto __pyx_L3; } - /* "View.MemoryView":989 + /* "View.MemoryView":991 * self.to_dtype_func(itemp, value) * else: * memoryview.assign_item_from_object(self, itemp, value) # <<<<<<<<<<<<<< @@ -17957,13 +18020,13 @@ static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memo * @property */ /*else*/ { - __pyx_t_3 = __pyx_memoryview_assign_item_from_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 989, __pyx_L1_error) + __pyx_t_3 = __pyx_memoryview_assign_item_from_object(((struct __pyx_memoryview_obj *)__pyx_v_self), __pyx_v_itemp, __pyx_v_value); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 991, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } __pyx_L3:; - /* "View.MemoryView":985 + /* "View.MemoryView":987 * return memoryview.convert_item_to_object(self, itemp) * * cdef assign_item_from_object(self, char *itemp, object value): # <<<<<<<<<<<<<< @@ -17984,7 +18047,7 @@ static PyObject *__pyx_memoryviewslice_assign_item_from_object(struct __pyx_memo return __pyx_r; } -/* "View.MemoryView":992 +/* "View.MemoryView":994 * * @property * def base(self): # <<<<<<<<<<<<<< @@ -18010,7 +18073,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_16_memoryviewslice_4base___get__ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - /* "View.MemoryView":993 + /* "View.MemoryView":995 * @property * def base(self): * return self.from_object # <<<<<<<<<<<<<< @@ -18022,7 +18085,7 @@ static PyObject *__pyx_pf_15View_dot_MemoryView_16_memoryviewslice_4base___get__ __pyx_r = __pyx_v_self->from_object; goto __pyx_L0; - /* "View.MemoryView":992 + /* "View.MemoryView":994 * * @property * def base(self): # <<<<<<<<<<<<<< @@ -18150,7 +18213,7 @@ static PyObject *__pyx_pf___pyx_memoryviewslice_2__setstate_cython__(CYTHON_UNUS return __pyx_r; } -/* "View.MemoryView":999 +/* "View.MemoryView":1001 * * @cname('__pyx_memoryview_fromslice') * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, # <<<<<<<<<<<<<< @@ -18178,7 +18241,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl int __pyx_clineno = 0; __Pyx_RefNannySetupContext("memoryview_fromslice", 0); - /* "View.MemoryView":1007 + /* "View.MemoryView":1009 * cdef _memoryviewslice result * * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< @@ -18188,7 +18251,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_t_1 = ((((PyObject *)__pyx_v_memviewslice.memview) == Py_None) != 0); if (__pyx_t_1) { - /* "View.MemoryView":1008 + /* "View.MemoryView":1010 * * if memviewslice.memview == Py_None: * return None # <<<<<<<<<<<<<< @@ -18199,7 +18262,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; - /* "View.MemoryView":1007 + /* "View.MemoryView":1009 * cdef _memoryviewslice result * * if memviewslice.memview == Py_None: # <<<<<<<<<<<<<< @@ -18208,16 +18271,16 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ } - /* "View.MemoryView":1013 + /* "View.MemoryView":1015 * * * result = _memoryviewslice(None, 0, dtype_is_object) # <<<<<<<<<<<<<< * * result.from_slice = memviewslice */ - __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1013, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_v_dtype_is_object); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1015, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1013, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1015, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(Py_None); __Pyx_GIVEREF(Py_None); @@ -18228,13 +18291,13 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_3, 2, __pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_memoryviewslice_type), __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1013, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_memoryviewslice_type), __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1015, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_v_result = ((struct __pyx_memoryviewslice_obj *)__pyx_t_2); __pyx_t_2 = 0; - /* "View.MemoryView":1015 + /* "View.MemoryView":1017 * result = _memoryviewslice(None, 0, dtype_is_object) * * result.from_slice = memviewslice # <<<<<<<<<<<<<< @@ -18243,7 +18306,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->from_slice = __pyx_v_memviewslice; - /* "View.MemoryView":1016 + /* "View.MemoryView":1018 * * result.from_slice = memviewslice * __PYX_INC_MEMVIEW(&memviewslice, 1) # <<<<<<<<<<<<<< @@ -18252,14 +18315,14 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __PYX_INC_MEMVIEW((&__pyx_v_memviewslice), 1); - /* "View.MemoryView":1018 + /* "View.MemoryView":1020 * __PYX_INC_MEMVIEW(&memviewslice, 1) * * result.from_object = ( memviewslice.memview).base # <<<<<<<<<<<<<< * result.typeinfo = memviewslice.memview.typeinfo * */ - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_memviewslice.memview), __pyx_n_s_base); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1018, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_v_memviewslice.memview), __pyx_n_s_base); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1020, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_2); __Pyx_GOTREF(__pyx_v_result->from_object); @@ -18267,7 +18330,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_v_result->from_object = __pyx_t_2; __pyx_t_2 = 0; - /* "View.MemoryView":1019 + /* "View.MemoryView":1021 * * result.from_object = ( memviewslice.memview).base * result.typeinfo = memviewslice.memview.typeinfo # <<<<<<<<<<<<<< @@ -18277,7 +18340,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_t_4 = __pyx_v_memviewslice.memview->typeinfo; __pyx_v_result->__pyx_base.typeinfo = __pyx_t_4; - /* "View.MemoryView":1021 + /* "View.MemoryView":1023 * result.typeinfo = memviewslice.memview.typeinfo * * result.view = memviewslice.memview.view # <<<<<<<<<<<<<< @@ -18287,7 +18350,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_t_5 = __pyx_v_memviewslice.memview->view; __pyx_v_result->__pyx_base.view = __pyx_t_5; - /* "View.MemoryView":1022 + /* "View.MemoryView":1024 * * result.view = memviewslice.memview.view * result.view.buf = memviewslice.data # <<<<<<<<<<<<<< @@ -18296,7 +18359,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->__pyx_base.view.buf = ((void *)__pyx_v_memviewslice.data); - /* "View.MemoryView":1023 + /* "View.MemoryView":1025 * result.view = memviewslice.memview.view * result.view.buf = memviewslice.data * result.view.ndim = ndim # <<<<<<<<<<<<<< @@ -18305,7 +18368,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->__pyx_base.view.ndim = __pyx_v_ndim; - /* "View.MemoryView":1024 + /* "View.MemoryView":1026 * result.view.buf = memviewslice.data * result.view.ndim = ndim * (<__pyx_buffer *> &result.view).obj = Py_None # <<<<<<<<<<<<<< @@ -18314,7 +18377,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ ((Py_buffer *)(&__pyx_v_result->__pyx_base.view))->obj = Py_None; - /* "View.MemoryView":1025 + /* "View.MemoryView":1027 * result.view.ndim = ndim * (<__pyx_buffer *> &result.view).obj = Py_None * Py_INCREF(Py_None) # <<<<<<<<<<<<<< @@ -18323,7 +18386,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ Py_INCREF(Py_None); - /* "View.MemoryView":1027 + /* "View.MemoryView":1029 * Py_INCREF(Py_None) * * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< @@ -18333,7 +18396,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_t_1 = ((((struct __pyx_memoryview_obj *)__pyx_v_memviewslice.memview)->flags & PyBUF_WRITABLE) != 0); if (__pyx_t_1) { - /* "View.MemoryView":1028 + /* "View.MemoryView":1030 * * if (memviewslice.memview).flags & PyBUF_WRITABLE: * result.flags = PyBUF_RECORDS # <<<<<<<<<<<<<< @@ -18342,7 +18405,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->__pyx_base.flags = PyBUF_RECORDS; - /* "View.MemoryView":1027 + /* "View.MemoryView":1029 * Py_INCREF(Py_None) * * if (memviewslice.memview).flags & PyBUF_WRITABLE: # <<<<<<<<<<<<<< @@ -18352,7 +18415,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl goto __pyx_L4; } - /* "View.MemoryView":1030 + /* "View.MemoryView":1032 * result.flags = PyBUF_RECORDS * else: * result.flags = PyBUF_RECORDS_RO # <<<<<<<<<<<<<< @@ -18364,7 +18427,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl } __pyx_L4:; - /* "View.MemoryView":1032 + /* "View.MemoryView":1034 * result.flags = PyBUF_RECORDS_RO * * result.view.shape = result.from_slice.shape # <<<<<<<<<<<<<< @@ -18373,7 +18436,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->__pyx_base.view.shape = ((Py_ssize_t *)__pyx_v_result->from_slice.shape); - /* "View.MemoryView":1033 + /* "View.MemoryView":1035 * * result.view.shape = result.from_slice.shape * result.view.strides = result.from_slice.strides # <<<<<<<<<<<<<< @@ -18382,7 +18445,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->__pyx_base.view.strides = ((Py_ssize_t *)__pyx_v_result->from_slice.strides); - /* "View.MemoryView":1036 + /* "View.MemoryView":1038 * * * result.view.suboffsets = NULL # <<<<<<<<<<<<<< @@ -18391,7 +18454,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->__pyx_base.view.suboffsets = NULL; - /* "View.MemoryView":1037 + /* "View.MemoryView":1039 * * result.view.suboffsets = NULL * for suboffset in result.from_slice.suboffsets[:ndim]: # <<<<<<<<<<<<<< @@ -18403,7 +18466,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_t_6 = __pyx_t_8; __pyx_v_suboffset = (__pyx_t_6[0]); - /* "View.MemoryView":1038 + /* "View.MemoryView":1040 * result.view.suboffsets = NULL * for suboffset in result.from_slice.suboffsets[:ndim]: * if suboffset >= 0: # <<<<<<<<<<<<<< @@ -18413,7 +18476,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_t_1 = ((__pyx_v_suboffset >= 0) != 0); if (__pyx_t_1) { - /* "View.MemoryView":1039 + /* "View.MemoryView":1041 * for suboffset in result.from_slice.suboffsets[:ndim]: * if suboffset >= 0: * result.view.suboffsets = result.from_slice.suboffsets # <<<<<<<<<<<<<< @@ -18422,7 +18485,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->__pyx_base.view.suboffsets = ((Py_ssize_t *)__pyx_v_result->from_slice.suboffsets); - /* "View.MemoryView":1040 + /* "View.MemoryView":1042 * if suboffset >= 0: * result.view.suboffsets = result.from_slice.suboffsets * break # <<<<<<<<<<<<<< @@ -18431,7 +18494,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ goto __pyx_L6_break; - /* "View.MemoryView":1038 + /* "View.MemoryView":1040 * result.view.suboffsets = NULL * for suboffset in result.from_slice.suboffsets[:ndim]: * if suboffset >= 0: # <<<<<<<<<<<<<< @@ -18442,7 +18505,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl } __pyx_L6_break:; - /* "View.MemoryView":1042 + /* "View.MemoryView":1044 * break * * result.view.len = result.view.itemsize # <<<<<<<<<<<<<< @@ -18452,7 +18515,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_t_9 = __pyx_v_result->__pyx_base.view.itemsize; __pyx_v_result->__pyx_base.view.len = __pyx_t_9; - /* "View.MemoryView":1043 + /* "View.MemoryView":1045 * * result.view.len = result.view.itemsize * for length in result.view.shape[:ndim]: # <<<<<<<<<<<<<< @@ -18462,29 +18525,29 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_t_7 = (__pyx_v_result->__pyx_base.view.shape + __pyx_v_ndim); for (__pyx_t_8 = __pyx_v_result->__pyx_base.view.shape; __pyx_t_8 < __pyx_t_7; __pyx_t_8++) { __pyx_t_6 = __pyx_t_8; - __pyx_t_2 = PyInt_FromSsize_t((__pyx_t_6[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1043, __pyx_L1_error) + __pyx_t_2 = PyInt_FromSsize_t((__pyx_t_6[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1045, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_XDECREF_SET(__pyx_v_length, __pyx_t_2); __pyx_t_2 = 0; - /* "View.MemoryView":1044 + /* "View.MemoryView":1046 * result.view.len = result.view.itemsize * for length in result.view.shape[:ndim]: * result.view.len *= length # <<<<<<<<<<<<<< * * result.to_object_func = to_object_func */ - __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_result->__pyx_base.view.len); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1044, __pyx_L1_error) + __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_result->__pyx_base.view.len); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1046, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyNumber_InPlaceMultiply(__pyx_t_2, __pyx_v_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1044, __pyx_L1_error) + __pyx_t_3 = PyNumber_InPlaceMultiply(__pyx_t_2, __pyx_v_length); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1046, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_3); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 1044, __pyx_L1_error) + __pyx_t_9 = __Pyx_PyIndex_AsSsize_t(__pyx_t_3); if (unlikely((__pyx_t_9 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(1, 1046, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_v_result->__pyx_base.view.len = __pyx_t_9; } - /* "View.MemoryView":1046 + /* "View.MemoryView":1048 * result.view.len *= length * * result.to_object_func = to_object_func # <<<<<<<<<<<<<< @@ -18493,7 +18556,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->to_object_func = __pyx_v_to_object_func; - /* "View.MemoryView":1047 + /* "View.MemoryView":1049 * * result.to_object_func = to_object_func * result.to_dtype_func = to_dtype_func # <<<<<<<<<<<<<< @@ -18502,7 +18565,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl */ __pyx_v_result->to_dtype_func = __pyx_v_to_dtype_func; - /* "View.MemoryView":1049 + /* "View.MemoryView":1051 * result.to_dtype_func = to_dtype_func * * return result # <<<<<<<<<<<<<< @@ -18514,7 +18577,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl __pyx_r = ((PyObject *)__pyx_v_result); goto __pyx_L0; - /* "View.MemoryView":999 + /* "View.MemoryView":1001 * * @cname('__pyx_memoryview_fromslice') * cdef memoryview_fromslice(__Pyx_memviewslice memviewslice, # <<<<<<<<<<<<<< @@ -18536,7 +18599,7 @@ static PyObject *__pyx_memoryview_fromslice(__Pyx_memviewslice __pyx_v_memviewsl return __pyx_r; } -/* "View.MemoryView":1052 +/* "View.MemoryView":1054 * * @cname('__pyx_memoryview_get_slice_from_memoryview') * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, # <<<<<<<<<<<<<< @@ -18556,7 +18619,7 @@ static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __p int __pyx_clineno = 0; __Pyx_RefNannySetupContext("get_slice_from_memview", 0); - /* "View.MemoryView":1055 + /* "View.MemoryView":1057 * __Pyx_memviewslice *mslice) except NULL: * cdef _memoryviewslice obj * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< @@ -18567,20 +18630,20 @@ static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __p __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "View.MemoryView":1056 + /* "View.MemoryView":1058 * cdef _memoryviewslice obj * if isinstance(memview, _memoryviewslice): * obj = memview # <<<<<<<<<<<<<< * return &obj.from_slice * else: */ - if (!(likely(((((PyObject *)__pyx_v_memview)) == Py_None) || likely(__Pyx_TypeTest(((PyObject *)__pyx_v_memview), __pyx_memoryviewslice_type))))) __PYX_ERR(1, 1056, __pyx_L1_error) + if (!(likely(((((PyObject *)__pyx_v_memview)) == Py_None) || likely(__Pyx_TypeTest(((PyObject *)__pyx_v_memview), __pyx_memoryviewslice_type))))) __PYX_ERR(1, 1058, __pyx_L1_error) __pyx_t_3 = ((PyObject *)__pyx_v_memview); __Pyx_INCREF(__pyx_t_3); __pyx_v_obj = ((struct __pyx_memoryviewslice_obj *)__pyx_t_3); __pyx_t_3 = 0; - /* "View.MemoryView":1057 + /* "View.MemoryView":1059 * if isinstance(memview, _memoryviewslice): * obj = memview * return &obj.from_slice # <<<<<<<<<<<<<< @@ -18590,7 +18653,7 @@ static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __p __pyx_r = (&__pyx_v_obj->from_slice); goto __pyx_L0; - /* "View.MemoryView":1055 + /* "View.MemoryView":1057 * __Pyx_memviewslice *mslice) except NULL: * cdef _memoryviewslice obj * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< @@ -18599,7 +18662,7 @@ static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __p */ } - /* "View.MemoryView":1059 + /* "View.MemoryView":1061 * return &obj.from_slice * else: * slice_copy(memview, mslice) # <<<<<<<<<<<<<< @@ -18609,7 +18672,7 @@ static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __p /*else*/ { __pyx_memoryview_slice_copy(__pyx_v_memview, __pyx_v_mslice); - /* "View.MemoryView":1060 + /* "View.MemoryView":1062 * else: * slice_copy(memview, mslice) * return mslice # <<<<<<<<<<<<<< @@ -18620,7 +18683,7 @@ static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __p goto __pyx_L0; } - /* "View.MemoryView":1052 + /* "View.MemoryView":1054 * * @cname('__pyx_memoryview_get_slice_from_memoryview') * cdef __Pyx_memviewslice *get_slice_from_memview(memoryview memview, # <<<<<<<<<<<<<< @@ -18639,7 +18702,7 @@ static __Pyx_memviewslice *__pyx_memoryview_get_slice_from_memoryview(struct __p return __pyx_r; } -/* "View.MemoryView":1063 +/* "View.MemoryView":1065 * * @cname('__pyx_memoryview_slice_copy') * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst): # <<<<<<<<<<<<<< @@ -18660,7 +18723,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem Py_ssize_t __pyx_t_5; __Pyx_RefNannySetupContext("slice_copy", 0); - /* "View.MemoryView":1067 + /* "View.MemoryView":1069 * cdef (Py_ssize_t*) shape, strides, suboffsets * * shape = memview.view.shape # <<<<<<<<<<<<<< @@ -18670,7 +18733,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem __pyx_t_1 = __pyx_v_memview->view.shape; __pyx_v_shape = __pyx_t_1; - /* "View.MemoryView":1068 + /* "View.MemoryView":1070 * * shape = memview.view.shape * strides = memview.view.strides # <<<<<<<<<<<<<< @@ -18680,7 +18743,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem __pyx_t_1 = __pyx_v_memview->view.strides; __pyx_v_strides = __pyx_t_1; - /* "View.MemoryView":1069 + /* "View.MemoryView":1071 * shape = memview.view.shape * strides = memview.view.strides * suboffsets = memview.view.suboffsets # <<<<<<<<<<<<<< @@ -18690,7 +18753,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem __pyx_t_1 = __pyx_v_memview->view.suboffsets; __pyx_v_suboffsets = __pyx_t_1; - /* "View.MemoryView":1071 + /* "View.MemoryView":1073 * suboffsets = memview.view.suboffsets * * dst.memview = <__pyx_memoryview *> memview # <<<<<<<<<<<<<< @@ -18699,7 +18762,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem */ __pyx_v_dst->memview = ((struct __pyx_memoryview_obj *)__pyx_v_memview); - /* "View.MemoryView":1072 + /* "View.MemoryView":1074 * * dst.memview = <__pyx_memoryview *> memview * dst.data = memview.view.buf # <<<<<<<<<<<<<< @@ -18708,7 +18771,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem */ __pyx_v_dst->data = ((char *)__pyx_v_memview->view.buf); - /* "View.MemoryView":1074 + /* "View.MemoryView":1076 * dst.data = memview.view.buf * * for dim in range(memview.view.ndim): # <<<<<<<<<<<<<< @@ -18720,7 +18783,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { __pyx_v_dim = __pyx_t_4; - /* "View.MemoryView":1075 + /* "View.MemoryView":1077 * * for dim in range(memview.view.ndim): * dst.shape[dim] = shape[dim] # <<<<<<<<<<<<<< @@ -18729,7 +18792,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem */ (__pyx_v_dst->shape[__pyx_v_dim]) = (__pyx_v_shape[__pyx_v_dim]); - /* "View.MemoryView":1076 + /* "View.MemoryView":1078 * for dim in range(memview.view.ndim): * dst.shape[dim] = shape[dim] * dst.strides[dim] = strides[dim] # <<<<<<<<<<<<<< @@ -18738,7 +18801,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem */ (__pyx_v_dst->strides[__pyx_v_dim]) = (__pyx_v_strides[__pyx_v_dim]); - /* "View.MemoryView":1077 + /* "View.MemoryView":1079 * dst.shape[dim] = shape[dim] * dst.strides[dim] = strides[dim] * dst.suboffsets[dim] = suboffsets[dim] if suboffsets else -1 # <<<<<<<<<<<<<< @@ -18753,7 +18816,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem (__pyx_v_dst->suboffsets[__pyx_v_dim]) = __pyx_t_5; } - /* "View.MemoryView":1063 + /* "View.MemoryView":1065 * * @cname('__pyx_memoryview_slice_copy') * cdef void slice_copy(memoryview memview, __Pyx_memviewslice *dst): # <<<<<<<<<<<<<< @@ -18765,7 +18828,7 @@ static void __pyx_memoryview_slice_copy(struct __pyx_memoryview_obj *__pyx_v_mem __Pyx_RefNannyFinishContext(); } -/* "View.MemoryView":1080 +/* "View.MemoryView":1082 * * @cname('__pyx_memoryview_copy_object') * cdef memoryview_copy(memoryview memview): # <<<<<<<<<<<<<< @@ -18783,7 +18846,7 @@ static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *__pyx int __pyx_clineno = 0; __Pyx_RefNannySetupContext("memoryview_copy", 0); - /* "View.MemoryView":1083 + /* "View.MemoryView":1085 * "Create a new memoryview object" * cdef __Pyx_memviewslice memviewslice * slice_copy(memview, &memviewslice) # <<<<<<<<<<<<<< @@ -18792,7 +18855,7 @@ static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *__pyx */ __pyx_memoryview_slice_copy(__pyx_v_memview, (&__pyx_v_memviewslice)); - /* "View.MemoryView":1084 + /* "View.MemoryView":1086 * cdef __Pyx_memviewslice memviewslice * slice_copy(memview, &memviewslice) * return memoryview_copy_from_slice(memview, &memviewslice) # <<<<<<<<<<<<<< @@ -18800,13 +18863,13 @@ static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *__pyx * @cname('__pyx_memoryview_copy_object_from_slice') */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_memoryview_copy_object_from_slice(__pyx_v_memview, (&__pyx_v_memviewslice)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1084, __pyx_L1_error) + __pyx_t_1 = __pyx_memoryview_copy_object_from_slice(__pyx_v_memview, (&__pyx_v_memviewslice)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1086, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "View.MemoryView":1080 + /* "View.MemoryView":1082 * * @cname('__pyx_memoryview_copy_object') * cdef memoryview_copy(memoryview memview): # <<<<<<<<<<<<<< @@ -18825,7 +18888,7 @@ static PyObject *__pyx_memoryview_copy_object(struct __pyx_memoryview_obj *__pyx return __pyx_r; } -/* "View.MemoryView":1087 +/* "View.MemoryView":1089 * * @cname('__pyx_memoryview_copy_object_from_slice') * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): # <<<<<<<<<<<<<< @@ -18848,7 +18911,7 @@ static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview int __pyx_clineno = 0; __Pyx_RefNannySetupContext("memoryview_copy_from_slice", 0); - /* "View.MemoryView":1094 + /* "View.MemoryView":1096 * cdef int (*to_dtype_func)(char *, object) except 0 * * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< @@ -18859,7 +18922,7 @@ static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "View.MemoryView":1095 + /* "View.MemoryView":1097 * * if isinstance(memview, _memoryviewslice): * to_object_func = (<_memoryviewslice> memview).to_object_func # <<<<<<<<<<<<<< @@ -18869,7 +18932,7 @@ static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview __pyx_t_3 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_object_func; __pyx_v_to_object_func = __pyx_t_3; - /* "View.MemoryView":1096 + /* "View.MemoryView":1098 * if isinstance(memview, _memoryviewslice): * to_object_func = (<_memoryviewslice> memview).to_object_func * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func # <<<<<<<<<<<<<< @@ -18879,7 +18942,7 @@ static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview __pyx_t_4 = ((struct __pyx_memoryviewslice_obj *)__pyx_v_memview)->to_dtype_func; __pyx_v_to_dtype_func = __pyx_t_4; - /* "View.MemoryView":1094 + /* "View.MemoryView":1096 * cdef int (*to_dtype_func)(char *, object) except 0 * * if isinstance(memview, _memoryviewslice): # <<<<<<<<<<<<<< @@ -18889,7 +18952,7 @@ static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview goto __pyx_L3; } - /* "View.MemoryView":1098 + /* "View.MemoryView":1100 * to_dtype_func = (<_memoryviewslice> memview).to_dtype_func * else: * to_object_func = NULL # <<<<<<<<<<<<<< @@ -18899,7 +18962,7 @@ static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview /*else*/ { __pyx_v_to_object_func = NULL; - /* "View.MemoryView":1099 + /* "View.MemoryView":1101 * else: * to_object_func = NULL * to_dtype_func = NULL # <<<<<<<<<<<<<< @@ -18910,7 +18973,7 @@ static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview } __pyx_L3:; - /* "View.MemoryView":1101 + /* "View.MemoryView":1103 * to_dtype_func = NULL * * return memoryview_fromslice(memviewslice[0], memview.view.ndim, # <<<<<<<<<<<<<< @@ -18919,20 +18982,20 @@ static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview */ __Pyx_XDECREF(__pyx_r); - /* "View.MemoryView":1103 + /* "View.MemoryView":1105 * return memoryview_fromslice(memviewslice[0], memview.view.ndim, * to_object_func, to_dtype_func, * memview.dtype_is_object) # <<<<<<<<<<<<<< * * */ - __pyx_t_5 = __pyx_memoryview_fromslice((__pyx_v_memviewslice[0]), __pyx_v_memview->view.ndim, __pyx_v_to_object_func, __pyx_v_to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 1101, __pyx_L1_error) + __pyx_t_5 = __pyx_memoryview_fromslice((__pyx_v_memviewslice[0]), __pyx_v_memview->view.ndim, __pyx_v_to_object_func, __pyx_v_to_dtype_func, __pyx_v_memview->dtype_is_object); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 1103, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_r = __pyx_t_5; __pyx_t_5 = 0; goto __pyx_L0; - /* "View.MemoryView":1087 + /* "View.MemoryView":1089 * * @cname('__pyx_memoryview_copy_object_from_slice') * cdef memoryview_copy_from_slice(memoryview memview, __Pyx_memviewslice *memviewslice): # <<<<<<<<<<<<<< @@ -18951,7 +19014,7 @@ static PyObject *__pyx_memoryview_copy_object_from_slice(struct __pyx_memoryview return __pyx_r; } -/* "View.MemoryView":1109 +/* "View.MemoryView":1111 * * * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) nogil: # <<<<<<<<<<<<<< @@ -18963,7 +19026,7 @@ static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { Py_ssize_t __pyx_r; int __pyx_t_1; - /* "View.MemoryView":1110 + /* "View.MemoryView":1112 * * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) nogil: * if arg < 0: # <<<<<<<<<<<<<< @@ -18973,7 +19036,7 @@ static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { __pyx_t_1 = ((__pyx_v_arg < 0) != 0); if (__pyx_t_1) { - /* "View.MemoryView":1111 + /* "View.MemoryView":1113 * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) nogil: * if arg < 0: * return -arg # <<<<<<<<<<<<<< @@ -18983,7 +19046,7 @@ static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { __pyx_r = (-__pyx_v_arg); goto __pyx_L0; - /* "View.MemoryView":1110 + /* "View.MemoryView":1112 * * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) nogil: * if arg < 0: # <<<<<<<<<<<<<< @@ -18992,7 +19055,7 @@ static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { */ } - /* "View.MemoryView":1113 + /* "View.MemoryView":1115 * return -arg * else: * return arg # <<<<<<<<<<<<<< @@ -19004,7 +19067,7 @@ static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { goto __pyx_L0; } - /* "View.MemoryView":1109 + /* "View.MemoryView":1111 * * * cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) nogil: # <<<<<<<<<<<<<< @@ -19017,7 +19080,7 @@ static Py_ssize_t abs_py_ssize_t(Py_ssize_t __pyx_v_arg) { return __pyx_r; } -/* "View.MemoryView":1116 +/* "View.MemoryView":1118 * * @cname('__pyx_get_best_slice_order') * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) nogil: # <<<<<<<<<<<<<< @@ -19035,7 +19098,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ int __pyx_t_3; int __pyx_t_4; - /* "View.MemoryView":1121 + /* "View.MemoryView":1123 * """ * cdef int i * cdef Py_ssize_t c_stride = 0 # <<<<<<<<<<<<<< @@ -19044,7 +19107,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ */ __pyx_v_c_stride = 0; - /* "View.MemoryView":1122 + /* "View.MemoryView":1124 * cdef int i * cdef Py_ssize_t c_stride = 0 * cdef Py_ssize_t f_stride = 0 # <<<<<<<<<<<<<< @@ -19053,7 +19116,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ */ __pyx_v_f_stride = 0; - /* "View.MemoryView":1124 + /* "View.MemoryView":1126 * cdef Py_ssize_t f_stride = 0 * * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< @@ -19063,7 +19126,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { __pyx_v_i = __pyx_t_1; - /* "View.MemoryView":1125 + /* "View.MemoryView":1127 * * for i in range(ndim - 1, -1, -1): * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< @@ -19073,7 +19136,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ __pyx_t_2 = (((__pyx_v_mslice->shape[__pyx_v_i]) > 1) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1126 + /* "View.MemoryView":1128 * for i in range(ndim - 1, -1, -1): * if mslice.shape[i] > 1: * c_stride = mslice.strides[i] # <<<<<<<<<<<<<< @@ -19082,7 +19145,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ */ __pyx_v_c_stride = (__pyx_v_mslice->strides[__pyx_v_i]); - /* "View.MemoryView":1127 + /* "View.MemoryView":1129 * if mslice.shape[i] > 1: * c_stride = mslice.strides[i] * break # <<<<<<<<<<<<<< @@ -19091,7 +19154,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ */ goto __pyx_L4_break; - /* "View.MemoryView":1125 + /* "View.MemoryView":1127 * * for i in range(ndim - 1, -1, -1): * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< @@ -19102,7 +19165,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ } __pyx_L4_break:; - /* "View.MemoryView":1129 + /* "View.MemoryView":1131 * break * * for i in range(ndim): # <<<<<<<<<<<<<< @@ -19114,7 +19177,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { __pyx_v_i = __pyx_t_4; - /* "View.MemoryView":1130 + /* "View.MemoryView":1132 * * for i in range(ndim): * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< @@ -19124,7 +19187,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ __pyx_t_2 = (((__pyx_v_mslice->shape[__pyx_v_i]) > 1) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1131 + /* "View.MemoryView":1133 * for i in range(ndim): * if mslice.shape[i] > 1: * f_stride = mslice.strides[i] # <<<<<<<<<<<<<< @@ -19133,7 +19196,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ */ __pyx_v_f_stride = (__pyx_v_mslice->strides[__pyx_v_i]); - /* "View.MemoryView":1132 + /* "View.MemoryView":1134 * if mslice.shape[i] > 1: * f_stride = mslice.strides[i] * break # <<<<<<<<<<<<<< @@ -19142,7 +19205,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ */ goto __pyx_L7_break; - /* "View.MemoryView":1130 + /* "View.MemoryView":1132 * * for i in range(ndim): * if mslice.shape[i] > 1: # <<<<<<<<<<<<<< @@ -19153,7 +19216,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ } __pyx_L7_break:; - /* "View.MemoryView":1134 + /* "View.MemoryView":1136 * break * * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< @@ -19163,7 +19226,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ __pyx_t_2 = ((abs_py_ssize_t(__pyx_v_c_stride) <= abs_py_ssize_t(__pyx_v_f_stride)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1135 + /* "View.MemoryView":1137 * * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): * return 'C' # <<<<<<<<<<<<<< @@ -19173,7 +19236,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ __pyx_r = 'C'; goto __pyx_L0; - /* "View.MemoryView":1134 + /* "View.MemoryView":1136 * break * * if abs_py_ssize_t(c_stride) <= abs_py_ssize_t(f_stride): # <<<<<<<<<<<<<< @@ -19182,7 +19245,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ */ } - /* "View.MemoryView":1137 + /* "View.MemoryView":1139 * return 'C' * else: * return 'F' # <<<<<<<<<<<<<< @@ -19194,7 +19257,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ goto __pyx_L0; } - /* "View.MemoryView":1116 + /* "View.MemoryView":1118 * * @cname('__pyx_get_best_slice_order') * cdef char get_best_order(__Pyx_memviewslice *mslice, int ndim) nogil: # <<<<<<<<<<<<<< @@ -19207,7 +19270,7 @@ static char __pyx_get_best_slice_order(__Pyx_memviewslice *__pyx_v_mslice, int _ return __pyx_r; } -/* "View.MemoryView":1140 +/* "View.MemoryView":1142 * * @cython.cdivision(True) * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, # <<<<<<<<<<<<<< @@ -19228,7 +19291,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v Py_ssize_t __pyx_t_5; Py_ssize_t __pyx_t_6; - /* "View.MemoryView":1147 + /* "View.MemoryView":1149 * * cdef Py_ssize_t i * cdef Py_ssize_t src_extent = src_shape[0] # <<<<<<<<<<<<<< @@ -19237,7 +19300,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ __pyx_v_src_extent = (__pyx_v_src_shape[0]); - /* "View.MemoryView":1148 + /* "View.MemoryView":1150 * cdef Py_ssize_t i * cdef Py_ssize_t src_extent = src_shape[0] * cdef Py_ssize_t dst_extent = dst_shape[0] # <<<<<<<<<<<<<< @@ -19246,7 +19309,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ __pyx_v_dst_extent = (__pyx_v_dst_shape[0]); - /* "View.MemoryView":1149 + /* "View.MemoryView":1151 * cdef Py_ssize_t src_extent = src_shape[0] * cdef Py_ssize_t dst_extent = dst_shape[0] * cdef Py_ssize_t src_stride = src_strides[0] # <<<<<<<<<<<<<< @@ -19255,7 +19318,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ __pyx_v_src_stride = (__pyx_v_src_strides[0]); - /* "View.MemoryView":1150 + /* "View.MemoryView":1152 * cdef Py_ssize_t dst_extent = dst_shape[0] * cdef Py_ssize_t src_stride = src_strides[0] * cdef Py_ssize_t dst_stride = dst_strides[0] # <<<<<<<<<<<<<< @@ -19264,7 +19327,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ __pyx_v_dst_stride = (__pyx_v_dst_strides[0]); - /* "View.MemoryView":1152 + /* "View.MemoryView":1154 * cdef Py_ssize_t dst_stride = dst_strides[0] * * if ndim == 1: # <<<<<<<<<<<<<< @@ -19274,7 +19337,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v __pyx_t_1 = ((__pyx_v_ndim == 1) != 0); if (__pyx_t_1) { - /* "View.MemoryView":1153 + /* "View.MemoryView":1155 * * if ndim == 1: * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< @@ -19294,7 +19357,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v goto __pyx_L5_bool_binop_done; } - /* "View.MemoryView":1154 + /* "View.MemoryView":1156 * if ndim == 1: * if (src_stride > 0 and dst_stride > 0 and * src_stride == itemsize == dst_stride): # <<<<<<<<<<<<<< @@ -19309,7 +19372,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v __pyx_t_1 = __pyx_t_3; __pyx_L5_bool_binop_done:; - /* "View.MemoryView":1153 + /* "View.MemoryView":1155 * * if ndim == 1: * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< @@ -19318,7 +19381,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ if (__pyx_t_1) { - /* "View.MemoryView":1155 + /* "View.MemoryView":1157 * if (src_stride > 0 and dst_stride > 0 and * src_stride == itemsize == dst_stride): * memcpy(dst_data, src_data, itemsize * dst_extent) # <<<<<<<<<<<<<< @@ -19327,7 +19390,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, (__pyx_v_itemsize * __pyx_v_dst_extent))); - /* "View.MemoryView":1153 + /* "View.MemoryView":1155 * * if ndim == 1: * if (src_stride > 0 and dst_stride > 0 and # <<<<<<<<<<<<<< @@ -19337,7 +19400,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v goto __pyx_L4; } - /* "View.MemoryView":1157 + /* "View.MemoryView":1159 * memcpy(dst_data, src_data, itemsize * dst_extent) * else: * for i in range(dst_extent): # <<<<<<<<<<<<<< @@ -19350,7 +19413,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { __pyx_v_i = __pyx_t_6; - /* "View.MemoryView":1158 + /* "View.MemoryView":1160 * else: * for i in range(dst_extent): * memcpy(dst_data, src_data, itemsize) # <<<<<<<<<<<<<< @@ -19359,7 +19422,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ (void)(memcpy(__pyx_v_dst_data, __pyx_v_src_data, __pyx_v_itemsize)); - /* "View.MemoryView":1159 + /* "View.MemoryView":1161 * for i in range(dst_extent): * memcpy(dst_data, src_data, itemsize) * src_data += src_stride # <<<<<<<<<<<<<< @@ -19368,7 +19431,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); - /* "View.MemoryView":1160 + /* "View.MemoryView":1162 * memcpy(dst_data, src_data, itemsize) * src_data += src_stride * dst_data += dst_stride # <<<<<<<<<<<<<< @@ -19380,7 +19443,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v } __pyx_L4:; - /* "View.MemoryView":1152 + /* "View.MemoryView":1154 * cdef Py_ssize_t dst_stride = dst_strides[0] * * if ndim == 1: # <<<<<<<<<<<<<< @@ -19390,7 +19453,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v goto __pyx_L3; } - /* "View.MemoryView":1162 + /* "View.MemoryView":1164 * dst_data += dst_stride * else: * for i in range(dst_extent): # <<<<<<<<<<<<<< @@ -19403,7 +19466,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { __pyx_v_i = __pyx_t_6; - /* "View.MemoryView":1163 + /* "View.MemoryView":1165 * else: * for i in range(dst_extent): * _copy_strided_to_strided(src_data, src_strides + 1, # <<<<<<<<<<<<<< @@ -19412,7 +19475,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ _copy_strided_to_strided(__pyx_v_src_data, (__pyx_v_src_strides + 1), __pyx_v_dst_data, (__pyx_v_dst_strides + 1), (__pyx_v_src_shape + 1), (__pyx_v_dst_shape + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize); - /* "View.MemoryView":1167 + /* "View.MemoryView":1169 * src_shape + 1, dst_shape + 1, * ndim - 1, itemsize) * src_data += src_stride # <<<<<<<<<<<<<< @@ -19421,7 +19484,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v */ __pyx_v_src_data = (__pyx_v_src_data + __pyx_v_src_stride); - /* "View.MemoryView":1168 + /* "View.MemoryView":1170 * ndim - 1, itemsize) * src_data += src_stride * dst_data += dst_stride # <<<<<<<<<<<<<< @@ -19433,7 +19496,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v } __pyx_L3:; - /* "View.MemoryView":1140 + /* "View.MemoryView":1142 * * @cython.cdivision(True) * cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, # <<<<<<<<<<<<<< @@ -19444,7 +19507,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v /* function exit code */ } -/* "View.MemoryView":1170 +/* "View.MemoryView":1172 * dst_data += dst_stride * * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< @@ -19454,7 +19517,7 @@ static void _copy_strided_to_strided(char *__pyx_v_src_data, Py_ssize_t *__pyx_v static void copy_strided_to_strided(__Pyx_memviewslice *__pyx_v_src, __Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize) { - /* "View.MemoryView":1173 + /* "View.MemoryView":1175 * __Pyx_memviewslice *dst, * int ndim, size_t itemsize) nogil: * _copy_strided_to_strided(src.data, src.strides, dst.data, dst.strides, # <<<<<<<<<<<<<< @@ -19463,7 +19526,7 @@ static void copy_strided_to_strided(__Pyx_memviewslice *__pyx_v_src, __Pyx_memvi */ _copy_strided_to_strided(__pyx_v_src->data, __pyx_v_src->strides, __pyx_v_dst->data, __pyx_v_dst->strides, __pyx_v_src->shape, __pyx_v_dst->shape, __pyx_v_ndim, __pyx_v_itemsize); - /* "View.MemoryView":1170 + /* "View.MemoryView":1172 * dst_data += dst_stride * * cdef void copy_strided_to_strided(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< @@ -19474,7 +19537,7 @@ static void copy_strided_to_strided(__Pyx_memviewslice *__pyx_v_src, __Pyx_memvi /* function exit code */ } -/* "View.MemoryView":1177 +/* "View.MemoryView":1179 * * @cname('__pyx_memoryview_slice_get_size') * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) nogil: # <<<<<<<<<<<<<< @@ -19491,7 +19554,7 @@ static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_sr Py_ssize_t *__pyx_t_3; Py_ssize_t *__pyx_t_4; - /* "View.MemoryView":1179 + /* "View.MemoryView":1181 * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) nogil: * "Return the size of the memory occupied by the slice in number of bytes" * cdef Py_ssize_t shape, size = src.memview.view.itemsize # <<<<<<<<<<<<<< @@ -19501,7 +19564,7 @@ static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_sr __pyx_t_1 = __pyx_v_src->memview->view.itemsize; __pyx_v_size = __pyx_t_1; - /* "View.MemoryView":1181 + /* "View.MemoryView":1183 * cdef Py_ssize_t shape, size = src.memview.view.itemsize * * for shape in src.shape[:ndim]: # <<<<<<<<<<<<<< @@ -19513,7 +19576,7 @@ static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_sr __pyx_t_2 = __pyx_t_4; __pyx_v_shape = (__pyx_t_2[0]); - /* "View.MemoryView":1182 + /* "View.MemoryView":1184 * * for shape in src.shape[:ndim]: * size *= shape # <<<<<<<<<<<<<< @@ -19523,7 +19586,7 @@ static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_sr __pyx_v_size = (__pyx_v_size * __pyx_v_shape); } - /* "View.MemoryView":1184 + /* "View.MemoryView":1186 * size *= shape * * return size # <<<<<<<<<<<<<< @@ -19533,7 +19596,7 @@ static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_sr __pyx_r = __pyx_v_size; goto __pyx_L0; - /* "View.MemoryView":1177 + /* "View.MemoryView":1179 * * @cname('__pyx_memoryview_slice_get_size') * cdef Py_ssize_t slice_get_size(__Pyx_memviewslice *src, int ndim) nogil: # <<<<<<<<<<<<<< @@ -19546,7 +19609,7 @@ static Py_ssize_t __pyx_memoryview_slice_get_size(__Pyx_memviewslice *__pyx_v_sr return __pyx_r; } -/* "View.MemoryView":1187 +/* "View.MemoryView":1189 * * @cname('__pyx_fill_contig_strides_array') * cdef Py_ssize_t fill_contig_strides_array( # <<<<<<<<<<<<<< @@ -19562,7 +19625,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ int __pyx_t_3; int __pyx_t_4; - /* "View.MemoryView":1196 + /* "View.MemoryView":1198 * cdef int idx * * if order == 'F': # <<<<<<<<<<<<<< @@ -19572,7 +19635,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ __pyx_t_1 = ((__pyx_v_order == 'F') != 0); if (__pyx_t_1) { - /* "View.MemoryView":1197 + /* "View.MemoryView":1199 * * if order == 'F': * for idx in range(ndim): # <<<<<<<<<<<<<< @@ -19584,7 +19647,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { __pyx_v_idx = __pyx_t_4; - /* "View.MemoryView":1198 + /* "View.MemoryView":1200 * if order == 'F': * for idx in range(ndim): * strides[idx] = stride # <<<<<<<<<<<<<< @@ -19593,7 +19656,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ */ (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; - /* "View.MemoryView":1199 + /* "View.MemoryView":1201 * for idx in range(ndim): * strides[idx] = stride * stride *= shape[idx] # <<<<<<<<<<<<<< @@ -19603,7 +19666,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ __pyx_v_stride = (__pyx_v_stride * (__pyx_v_shape[__pyx_v_idx])); } - /* "View.MemoryView":1196 + /* "View.MemoryView":1198 * cdef int idx * * if order == 'F': # <<<<<<<<<<<<<< @@ -19613,7 +19676,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ goto __pyx_L3; } - /* "View.MemoryView":1201 + /* "View.MemoryView":1203 * stride *= shape[idx] * else: * for idx in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< @@ -19624,7 +19687,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ for (__pyx_t_2 = (__pyx_v_ndim - 1); __pyx_t_2 > -1; __pyx_t_2-=1) { __pyx_v_idx = __pyx_t_2; - /* "View.MemoryView":1202 + /* "View.MemoryView":1204 * else: * for idx in range(ndim - 1, -1, -1): * strides[idx] = stride # <<<<<<<<<<<<<< @@ -19633,7 +19696,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ */ (__pyx_v_strides[__pyx_v_idx]) = __pyx_v_stride; - /* "View.MemoryView":1203 + /* "View.MemoryView":1205 * for idx in range(ndim - 1, -1, -1): * strides[idx] = stride * stride *= shape[idx] # <<<<<<<<<<<<<< @@ -19645,7 +19708,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ } __pyx_L3:; - /* "View.MemoryView":1205 + /* "View.MemoryView":1207 * stride *= shape[idx] * * return stride # <<<<<<<<<<<<<< @@ -19655,7 +19718,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ __pyx_r = __pyx_v_stride; goto __pyx_L0; - /* "View.MemoryView":1187 + /* "View.MemoryView":1189 * * @cname('__pyx_fill_contig_strides_array') * cdef Py_ssize_t fill_contig_strides_array( # <<<<<<<<<<<<<< @@ -19668,7 +19731,7 @@ static Py_ssize_t __pyx_fill_contig_strides_array(Py_ssize_t *__pyx_v_shape, Py_ return __pyx_r; } -/* "View.MemoryView":1208 +/* "View.MemoryView":1210 * * @cname('__pyx_memoryview_copy_data_to_temp') * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< @@ -19692,7 +19755,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, const char *__pyx_filename = NULL; int __pyx_clineno = 0; - /* "View.MemoryView":1219 + /* "View.MemoryView":1221 * cdef void *result * * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< @@ -19702,7 +19765,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __pyx_t_1 = __pyx_v_src->memview->view.itemsize; __pyx_v_itemsize = __pyx_t_1; - /* "View.MemoryView":1220 + /* "View.MemoryView":1222 * * cdef size_t itemsize = src.memview.view.itemsize * cdef size_t size = slice_get_size(src, ndim) # <<<<<<<<<<<<<< @@ -19711,7 +19774,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, */ __pyx_v_size = __pyx_memoryview_slice_get_size(__pyx_v_src, __pyx_v_ndim); - /* "View.MemoryView":1222 + /* "View.MemoryView":1224 * cdef size_t size = slice_get_size(src, ndim) * * result = malloc(size) # <<<<<<<<<<<<<< @@ -19720,7 +19783,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, */ __pyx_v_result = malloc(__pyx_v_size); - /* "View.MemoryView":1223 + /* "View.MemoryView":1225 * * result = malloc(size) * if not result: # <<<<<<<<<<<<<< @@ -19730,16 +19793,16 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __pyx_t_2 = ((!(__pyx_v_result != 0)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1224 + /* "View.MemoryView":1226 * result = malloc(size) * if not result: * _err(MemoryError, NULL) # <<<<<<<<<<<<<< * * */ - __pyx_t_3 = __pyx_memoryview_err(__pyx_builtin_MemoryError, NULL); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 1224, __pyx_L1_error) + __pyx_t_3 = __pyx_memoryview_err(__pyx_builtin_MemoryError, NULL); if (unlikely(__pyx_t_3 == ((int)-1))) __PYX_ERR(1, 1226, __pyx_L1_error) - /* "View.MemoryView":1223 + /* "View.MemoryView":1225 * * result = malloc(size) * if not result: # <<<<<<<<<<<<<< @@ -19748,7 +19811,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, */ } - /* "View.MemoryView":1227 + /* "View.MemoryView":1229 * * * tmpslice.data = result # <<<<<<<<<<<<<< @@ -19757,7 +19820,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, */ __pyx_v_tmpslice->data = ((char *)__pyx_v_result); - /* "View.MemoryView":1228 + /* "View.MemoryView":1230 * * tmpslice.data = result * tmpslice.memview = src.memview # <<<<<<<<<<<<<< @@ -19767,7 +19830,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __pyx_t_4 = __pyx_v_src->memview; __pyx_v_tmpslice->memview = __pyx_t_4; - /* "View.MemoryView":1229 + /* "View.MemoryView":1231 * tmpslice.data = result * tmpslice.memview = src.memview * for i in range(ndim): # <<<<<<<<<<<<<< @@ -19779,7 +19842,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { __pyx_v_i = __pyx_t_6; - /* "View.MemoryView":1230 + /* "View.MemoryView":1232 * tmpslice.memview = src.memview * for i in range(ndim): * tmpslice.shape[i] = src.shape[i] # <<<<<<<<<<<<<< @@ -19788,7 +19851,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, */ (__pyx_v_tmpslice->shape[__pyx_v_i]) = (__pyx_v_src->shape[__pyx_v_i]); - /* "View.MemoryView":1231 + /* "View.MemoryView":1233 * for i in range(ndim): * tmpslice.shape[i] = src.shape[i] * tmpslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< @@ -19798,7 +19861,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, (__pyx_v_tmpslice->suboffsets[__pyx_v_i]) = -1L; } - /* "View.MemoryView":1233 + /* "View.MemoryView":1235 * tmpslice.suboffsets[i] = -1 * * fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, # <<<<<<<<<<<<<< @@ -19807,7 +19870,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, */ (void)(__pyx_fill_contig_strides_array((&(__pyx_v_tmpslice->shape[0])), (&(__pyx_v_tmpslice->strides[0])), __pyx_v_itemsize, __pyx_v_ndim, __pyx_v_order)); - /* "View.MemoryView":1237 + /* "View.MemoryView":1239 * * * for i in range(ndim): # <<<<<<<<<<<<<< @@ -19819,7 +19882,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { __pyx_v_i = __pyx_t_6; - /* "View.MemoryView":1238 + /* "View.MemoryView":1240 * * for i in range(ndim): * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< @@ -19829,7 +19892,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __pyx_t_2 = (((__pyx_v_tmpslice->shape[__pyx_v_i]) == 1) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1239 + /* "View.MemoryView":1241 * for i in range(ndim): * if tmpslice.shape[i] == 1: * tmpslice.strides[i] = 0 # <<<<<<<<<<<<<< @@ -19838,7 +19901,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, */ (__pyx_v_tmpslice->strides[__pyx_v_i]) = 0; - /* "View.MemoryView":1238 + /* "View.MemoryView":1240 * * for i in range(ndim): * if tmpslice.shape[i] == 1: # <<<<<<<<<<<<<< @@ -19848,7 +19911,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, } } - /* "View.MemoryView":1241 + /* "View.MemoryView":1243 * tmpslice.strides[i] = 0 * * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< @@ -19858,7 +19921,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __pyx_t_2 = (__pyx_memviewslice_is_contig((__pyx_v_src[0]), __pyx_v_order, __pyx_v_ndim) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1242 + /* "View.MemoryView":1244 * * if slice_is_contig(src[0], order, ndim): * memcpy(result, src.data, size) # <<<<<<<<<<<<<< @@ -19867,7 +19930,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, */ (void)(memcpy(__pyx_v_result, __pyx_v_src->data, __pyx_v_size)); - /* "View.MemoryView":1241 + /* "View.MemoryView":1243 * tmpslice.strides[i] = 0 * * if slice_is_contig(src[0], order, ndim): # <<<<<<<<<<<<<< @@ -19877,7 +19940,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, goto __pyx_L9; } - /* "View.MemoryView":1244 + /* "View.MemoryView":1246 * memcpy(result, src.data, size) * else: * copy_strided_to_strided(src, tmpslice, ndim, itemsize) # <<<<<<<<<<<<<< @@ -19889,7 +19952,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, } __pyx_L9:; - /* "View.MemoryView":1246 + /* "View.MemoryView":1248 * copy_strided_to_strided(src, tmpslice, ndim, itemsize) * * return result # <<<<<<<<<<<<<< @@ -19899,7 +19962,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, __pyx_r = __pyx_v_result; goto __pyx_L0; - /* "View.MemoryView":1208 + /* "View.MemoryView":1210 * * @cname('__pyx_memoryview_copy_data_to_temp') * cdef void *copy_data_to_temp(__Pyx_memviewslice *src, # <<<<<<<<<<<<<< @@ -19923,7 +19986,7 @@ static void *__pyx_memoryview_copy_data_to_temp(__Pyx_memviewslice *__pyx_v_src, return __pyx_r; } -/* "View.MemoryView":1251 +/* "View.MemoryView":1253 * * @cname('__pyx_memoryview_err_extents') * cdef int _err_extents(int i, Py_ssize_t extent1, # <<<<<<<<<<<<<< @@ -19946,20 +20009,20 @@ static int __pyx_memoryview_err_extents(int __pyx_v_i, Py_ssize_t __pyx_v_extent #endif __Pyx_RefNannySetupContext("_err_extents", 0); - /* "View.MemoryView":1254 + /* "View.MemoryView":1256 * Py_ssize_t extent2) except -1 with gil: * raise ValueError("got differing extents in dimension %d (got %d and %d)" % * (i, extent1, extent2)) # <<<<<<<<<<<<<< * * @cname('__pyx_memoryview_err_dim') */ - __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_i); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1254, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_i); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1256, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_extent1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1254, __pyx_L1_error) + __pyx_t_2 = PyInt_FromSsize_t(__pyx_v_extent1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1256, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyInt_FromSsize_t(__pyx_v_extent2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1254, __pyx_L1_error) + __pyx_t_3 = PyInt_FromSsize_t(__pyx_v_extent2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1256, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyTuple_New(3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1254, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1256, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_1); @@ -19971,24 +20034,24 @@ static int __pyx_memoryview_err_extents(int __pyx_v_i, Py_ssize_t __pyx_v_extent __pyx_t_2 = 0; __pyx_t_3 = 0; - /* "View.MemoryView":1253 + /* "View.MemoryView":1255 * cdef int _err_extents(int i, Py_ssize_t extent1, * Py_ssize_t extent2) except -1 with gil: * raise ValueError("got differing extents in dimension %d (got %d and %d)" % # <<<<<<<<<<<<<< * (i, extent1, extent2)) * */ - __pyx_t_3 = __Pyx_PyString_Format(__pyx_kp_s_got_differing_extents_in_dimensi, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1253, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyString_Format(__pyx_kp_s_got_differing_extents_in_dimensi, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1255, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1253, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_builtin_ValueError, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1255, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 1253, __pyx_L1_error) + __PYX_ERR(1, 1255, __pyx_L1_error) - /* "View.MemoryView":1251 + /* "View.MemoryView":1253 * * @cname('__pyx_memoryview_err_extents') * cdef int _err_extents(int i, Py_ssize_t extent1, # <<<<<<<<<<<<<< @@ -20011,7 +20074,7 @@ static int __pyx_memoryview_err_extents(int __pyx_v_i, Py_ssize_t __pyx_v_extent return __pyx_r; } -/* "View.MemoryView":1257 +/* "View.MemoryView":1259 * * @cname('__pyx_memoryview_err_dim') * cdef int _err_dim(object error, char *msg, int dim) except -1 with gil: # <<<<<<<<<<<<<< @@ -20035,18 +20098,18 @@ static int __pyx_memoryview_err_dim(PyObject *__pyx_v_error, char *__pyx_v_msg, __Pyx_RefNannySetupContext("_err_dim", 0); __Pyx_INCREF(__pyx_v_error); - /* "View.MemoryView":1258 + /* "View.MemoryView":1260 * @cname('__pyx_memoryview_err_dim') * cdef int _err_dim(object error, char *msg, int dim) except -1 with gil: * raise error(msg.decode('ascii') % dim) # <<<<<<<<<<<<<< * * @cname('__pyx_memoryview_err') */ - __pyx_t_2 = __Pyx_decode_c_string(__pyx_v_msg, 0, strlen(__pyx_v_msg), NULL, NULL, PyUnicode_DecodeASCII); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1258, __pyx_L1_error) + __pyx_t_2 = __Pyx_decode_c_string(__pyx_v_msg, 0, strlen(__pyx_v_msg), NULL, NULL, PyUnicode_DecodeASCII); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1260, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_dim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1258, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_dim); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1260, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = PyUnicode_Format(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1258, __pyx_L1_error) + __pyx_t_4 = PyUnicode_Format(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 1260, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -20064,14 +20127,14 @@ static int __pyx_memoryview_err_dim(PyObject *__pyx_v_error, char *__pyx_v_msg, __pyx_t_1 = (__pyx_t_2) ? __Pyx_PyObject_Call2Args(__pyx_t_3, __pyx_t_2, __pyx_t_4) : __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_4); __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1258, __pyx_L1_error) + if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1260, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(1, 1258, __pyx_L1_error) + __PYX_ERR(1, 1260, __pyx_L1_error) - /* "View.MemoryView":1257 + /* "View.MemoryView":1259 * * @cname('__pyx_memoryview_err_dim') * cdef int _err_dim(object error, char *msg, int dim) except -1 with gil: # <<<<<<<<<<<<<< @@ -20095,7 +20158,7 @@ static int __pyx_memoryview_err_dim(PyObject *__pyx_v_error, char *__pyx_v_msg, return __pyx_r; } -/* "View.MemoryView":1261 +/* "View.MemoryView":1263 * * @cname('__pyx_memoryview_err') * cdef int _err(object error, char *msg) except -1 with gil: # <<<<<<<<<<<<<< @@ -20120,7 +20183,7 @@ static int __pyx_memoryview_err(PyObject *__pyx_v_error, char *__pyx_v_msg) { __Pyx_RefNannySetupContext("_err", 0); __Pyx_INCREF(__pyx_v_error); - /* "View.MemoryView":1262 + /* "View.MemoryView":1264 * @cname('__pyx_memoryview_err') * cdef int _err(object error, char *msg) except -1 with gil: * if msg != NULL: # <<<<<<<<<<<<<< @@ -20130,14 +20193,14 @@ static int __pyx_memoryview_err(PyObject *__pyx_v_error, char *__pyx_v_msg) { __pyx_t_1 = ((__pyx_v_msg != NULL) != 0); if (unlikely(__pyx_t_1)) { - /* "View.MemoryView":1263 + /* "View.MemoryView":1265 * cdef int _err(object error, char *msg) except -1 with gil: * if msg != NULL: * raise error(msg.decode('ascii')) # <<<<<<<<<<<<<< * else: * raise error */ - __pyx_t_3 = __Pyx_decode_c_string(__pyx_v_msg, 0, strlen(__pyx_v_msg), NULL, NULL, PyUnicode_DecodeASCII); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1263, __pyx_L1_error) + __pyx_t_3 = __Pyx_decode_c_string(__pyx_v_msg, 0, strlen(__pyx_v_msg), NULL, NULL, PyUnicode_DecodeASCII); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 1265, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(__pyx_v_error); __pyx_t_4 = __pyx_v_error; __pyx_t_5 = NULL; @@ -20153,14 +20216,14 @@ static int __pyx_memoryview_err(PyObject *__pyx_v_error, char *__pyx_v_msg) { __pyx_t_2 = (__pyx_t_5) ? __Pyx_PyObject_Call2Args(__pyx_t_4, __pyx_t_5, __pyx_t_3) : __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1263, __pyx_L1_error) + if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 1265, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 1263, __pyx_L1_error) + __PYX_ERR(1, 1265, __pyx_L1_error) - /* "View.MemoryView":1262 + /* "View.MemoryView":1264 * @cname('__pyx_memoryview_err') * cdef int _err(object error, char *msg) except -1 with gil: * if msg != NULL: # <<<<<<<<<<<<<< @@ -20169,7 +20232,7 @@ static int __pyx_memoryview_err(PyObject *__pyx_v_error, char *__pyx_v_msg) { */ } - /* "View.MemoryView":1265 + /* "View.MemoryView":1267 * raise error(msg.decode('ascii')) * else: * raise error # <<<<<<<<<<<<<< @@ -20178,10 +20241,10 @@ static int __pyx_memoryview_err(PyObject *__pyx_v_error, char *__pyx_v_msg) { */ /*else*/ { __Pyx_Raise(__pyx_v_error, 0, 0, 0); - __PYX_ERR(1, 1265, __pyx_L1_error) + __PYX_ERR(1, 1267, __pyx_L1_error) } - /* "View.MemoryView":1261 + /* "View.MemoryView":1263 * * @cname('__pyx_memoryview_err') * cdef int _err(object error, char *msg) except -1 with gil: # <<<<<<<<<<<<<< @@ -20205,7 +20268,7 @@ static int __pyx_memoryview_err(PyObject *__pyx_v_error, char *__pyx_v_msg) { return __pyx_r; } -/* "View.MemoryView":1268 +/* "View.MemoryView":1270 * * @cname('__pyx_memoryview_copy_contents') * cdef int memoryview_copy_contents(__Pyx_memviewslice src, # <<<<<<<<<<<<<< @@ -20235,7 +20298,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ const char *__pyx_filename = NULL; int __pyx_clineno = 0; - /* "View.MemoryView":1276 + /* "View.MemoryView":1278 * Check for overlapping memory and verify the shapes. * """ * cdef void *tmpdata = NULL # <<<<<<<<<<<<<< @@ -20244,7 +20307,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_v_tmpdata = NULL; - /* "View.MemoryView":1277 + /* "View.MemoryView":1279 * """ * cdef void *tmpdata = NULL * cdef size_t itemsize = src.memview.view.itemsize # <<<<<<<<<<<<<< @@ -20254,7 +20317,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_1 = __pyx_v_src.memview->view.itemsize; __pyx_v_itemsize = __pyx_t_1; - /* "View.MemoryView":1279 + /* "View.MemoryView":1281 * cdef size_t itemsize = src.memview.view.itemsize * cdef int i * cdef char order = get_best_order(&src, src_ndim) # <<<<<<<<<<<<<< @@ -20263,7 +20326,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_src), __pyx_v_src_ndim); - /* "View.MemoryView":1280 + /* "View.MemoryView":1282 * cdef int i * cdef char order = get_best_order(&src, src_ndim) * cdef bint broadcasting = False # <<<<<<<<<<<<<< @@ -20272,7 +20335,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_v_broadcasting = 0; - /* "View.MemoryView":1281 + /* "View.MemoryView":1283 * cdef char order = get_best_order(&src, src_ndim) * cdef bint broadcasting = False * cdef bint direct_copy = False # <<<<<<<<<<<<<< @@ -20281,7 +20344,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_v_direct_copy = 0; - /* "View.MemoryView":1284 + /* "View.MemoryView":1286 * cdef __Pyx_memviewslice tmp * * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< @@ -20291,7 +20354,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = ((__pyx_v_src_ndim < __pyx_v_dst_ndim) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1285 + /* "View.MemoryView":1287 * * if src_ndim < dst_ndim: * broadcast_leading(&src, src_ndim, dst_ndim) # <<<<<<<<<<<<<< @@ -20300,7 +20363,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_memoryview_broadcast_leading((&__pyx_v_src), __pyx_v_src_ndim, __pyx_v_dst_ndim); - /* "View.MemoryView":1284 + /* "View.MemoryView":1286 * cdef __Pyx_memviewslice tmp * * if src_ndim < dst_ndim: # <<<<<<<<<<<<<< @@ -20310,7 +20373,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ goto __pyx_L3; } - /* "View.MemoryView":1286 + /* "View.MemoryView":1288 * if src_ndim < dst_ndim: * broadcast_leading(&src, src_ndim, dst_ndim) * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< @@ -20320,7 +20383,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = ((__pyx_v_dst_ndim < __pyx_v_src_ndim) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1287 + /* "View.MemoryView":1289 * broadcast_leading(&src, src_ndim, dst_ndim) * elif dst_ndim < src_ndim: * broadcast_leading(&dst, dst_ndim, src_ndim) # <<<<<<<<<<<<<< @@ -20329,7 +20392,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_memoryview_broadcast_leading((&__pyx_v_dst), __pyx_v_dst_ndim, __pyx_v_src_ndim); - /* "View.MemoryView":1286 + /* "View.MemoryView":1288 * if src_ndim < dst_ndim: * broadcast_leading(&src, src_ndim, dst_ndim) * elif dst_ndim < src_ndim: # <<<<<<<<<<<<<< @@ -20339,7 +20402,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ } __pyx_L3:; - /* "View.MemoryView":1289 + /* "View.MemoryView":1291 * broadcast_leading(&dst, dst_ndim, src_ndim) * * cdef int ndim = max(src_ndim, dst_ndim) # <<<<<<<<<<<<<< @@ -20355,7 +20418,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ } __pyx_v_ndim = __pyx_t_5; - /* "View.MemoryView":1291 + /* "View.MemoryView":1293 * cdef int ndim = max(src_ndim, dst_ndim) * * for i in range(ndim): # <<<<<<<<<<<<<< @@ -20367,7 +20430,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { __pyx_v_i = __pyx_t_4; - /* "View.MemoryView":1292 + /* "View.MemoryView":1294 * * for i in range(ndim): * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< @@ -20377,7 +20440,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = (((__pyx_v_src.shape[__pyx_v_i]) != (__pyx_v_dst.shape[__pyx_v_i])) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1293 + /* "View.MemoryView":1295 * for i in range(ndim): * if src.shape[i] != dst.shape[i]: * if src.shape[i] == 1: # <<<<<<<<<<<<<< @@ -20387,7 +20450,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = (((__pyx_v_src.shape[__pyx_v_i]) == 1) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1294 + /* "View.MemoryView":1296 * if src.shape[i] != dst.shape[i]: * if src.shape[i] == 1: * broadcasting = True # <<<<<<<<<<<<<< @@ -20396,7 +20459,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_v_broadcasting = 1; - /* "View.MemoryView":1295 + /* "View.MemoryView":1297 * if src.shape[i] == 1: * broadcasting = True * src.strides[i] = 0 # <<<<<<<<<<<<<< @@ -20405,7 +20468,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ (__pyx_v_src.strides[__pyx_v_i]) = 0; - /* "View.MemoryView":1293 + /* "View.MemoryView":1295 * for i in range(ndim): * if src.shape[i] != dst.shape[i]: * if src.shape[i] == 1: # <<<<<<<<<<<<<< @@ -20415,7 +20478,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ goto __pyx_L7; } - /* "View.MemoryView":1297 + /* "View.MemoryView":1299 * src.strides[i] = 0 * else: * _err_extents(i, dst.shape[i], src.shape[i]) # <<<<<<<<<<<<<< @@ -20423,11 +20486,11 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ * if src.suboffsets[i] >= 0: */ /*else*/ { - __pyx_t_6 = __pyx_memoryview_err_extents(__pyx_v_i, (__pyx_v_dst.shape[__pyx_v_i]), (__pyx_v_src.shape[__pyx_v_i])); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1297, __pyx_L1_error) + __pyx_t_6 = __pyx_memoryview_err_extents(__pyx_v_i, (__pyx_v_dst.shape[__pyx_v_i]), (__pyx_v_src.shape[__pyx_v_i])); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1299, __pyx_L1_error) } __pyx_L7:; - /* "View.MemoryView":1292 + /* "View.MemoryView":1294 * * for i in range(ndim): * if src.shape[i] != dst.shape[i]: # <<<<<<<<<<<<<< @@ -20436,7 +20499,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ } - /* "View.MemoryView":1299 + /* "View.MemoryView":1301 * _err_extents(i, dst.shape[i], src.shape[i]) * * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< @@ -20446,16 +20509,16 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = (((__pyx_v_src.suboffsets[__pyx_v_i]) >= 0) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1300 + /* "View.MemoryView":1302 * * if src.suboffsets[i] >= 0: * _err_dim(ValueError, "Dimension %d is not direct", i) # <<<<<<<<<<<<<< * * if slices_overlap(&src, &dst, ndim, itemsize): */ - __pyx_t_6 = __pyx_memoryview_err_dim(__pyx_builtin_ValueError, ((char *)"Dimension %d is not direct"), __pyx_v_i); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1300, __pyx_L1_error) + __pyx_t_6 = __pyx_memoryview_err_dim(__pyx_builtin_ValueError, ((char *)"Dimension %d is not direct"), __pyx_v_i); if (unlikely(__pyx_t_6 == ((int)-1))) __PYX_ERR(1, 1302, __pyx_L1_error) - /* "View.MemoryView":1299 + /* "View.MemoryView":1301 * _err_extents(i, dst.shape[i], src.shape[i]) * * if src.suboffsets[i] >= 0: # <<<<<<<<<<<<<< @@ -20465,7 +20528,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ } } - /* "View.MemoryView":1302 + /* "View.MemoryView":1304 * _err_dim(ValueError, "Dimension %d is not direct", i) * * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< @@ -20475,7 +20538,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = (__pyx_slices_overlap((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1304 + /* "View.MemoryView":1306 * if slices_overlap(&src, &dst, ndim, itemsize): * * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< @@ -20485,7 +20548,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = ((!(__pyx_memviewslice_is_contig(__pyx_v_src, __pyx_v_order, __pyx_v_ndim) != 0)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1305 + /* "View.MemoryView":1307 * * if not slice_is_contig(src, order, ndim): * order = get_best_order(&dst, ndim) # <<<<<<<<<<<<<< @@ -20494,7 +20557,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_v_order = __pyx_get_best_slice_order((&__pyx_v_dst), __pyx_v_ndim); - /* "View.MemoryView":1304 + /* "View.MemoryView":1306 * if slices_overlap(&src, &dst, ndim, itemsize): * * if not slice_is_contig(src, order, ndim): # <<<<<<<<<<<<<< @@ -20503,17 +20566,17 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ } - /* "View.MemoryView":1307 + /* "View.MemoryView":1309 * order = get_best_order(&dst, ndim) * * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) # <<<<<<<<<<<<<< * src = tmp * */ - __pyx_t_7 = __pyx_memoryview_copy_data_to_temp((&__pyx_v_src), (&__pyx_v_tmp), __pyx_v_order, __pyx_v_ndim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 1307, __pyx_L1_error) + __pyx_t_7 = __pyx_memoryview_copy_data_to_temp((&__pyx_v_src), (&__pyx_v_tmp), __pyx_v_order, __pyx_v_ndim); if (unlikely(__pyx_t_7 == ((void *)NULL))) __PYX_ERR(1, 1309, __pyx_L1_error) __pyx_v_tmpdata = __pyx_t_7; - /* "View.MemoryView":1308 + /* "View.MemoryView":1310 * * tmpdata = copy_data_to_temp(&src, &tmp, order, ndim) * src = tmp # <<<<<<<<<<<<<< @@ -20522,7 +20585,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_v_src = __pyx_v_tmp; - /* "View.MemoryView":1302 + /* "View.MemoryView":1304 * _err_dim(ValueError, "Dimension %d is not direct", i) * * if slices_overlap(&src, &dst, ndim, itemsize): # <<<<<<<<<<<<<< @@ -20531,7 +20594,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ } - /* "View.MemoryView":1310 + /* "View.MemoryView":1312 * src = tmp * * if not broadcasting: # <<<<<<<<<<<<<< @@ -20541,7 +20604,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = ((!(__pyx_v_broadcasting != 0)) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1313 + /* "View.MemoryView":1315 * * * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< @@ -20551,7 +20614,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = (__pyx_memviewslice_is_contig(__pyx_v_src, 'C', __pyx_v_ndim) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1314 + /* "View.MemoryView":1316 * * if slice_is_contig(src, 'C', ndim): * direct_copy = slice_is_contig(dst, 'C', ndim) # <<<<<<<<<<<<<< @@ -20560,7 +20623,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'C', __pyx_v_ndim); - /* "View.MemoryView":1313 + /* "View.MemoryView":1315 * * * if slice_is_contig(src, 'C', ndim): # <<<<<<<<<<<<<< @@ -20570,7 +20633,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ goto __pyx_L12; } - /* "View.MemoryView":1315 + /* "View.MemoryView":1317 * if slice_is_contig(src, 'C', ndim): * direct_copy = slice_is_contig(dst, 'C', ndim) * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< @@ -20580,7 +20643,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = (__pyx_memviewslice_is_contig(__pyx_v_src, 'F', __pyx_v_ndim) != 0); if (__pyx_t_2) { - /* "View.MemoryView":1316 + /* "View.MemoryView":1318 * direct_copy = slice_is_contig(dst, 'C', ndim) * elif slice_is_contig(src, 'F', ndim): * direct_copy = slice_is_contig(dst, 'F', ndim) # <<<<<<<<<<<<<< @@ -20589,7 +20652,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_v_direct_copy = __pyx_memviewslice_is_contig(__pyx_v_dst, 'F', __pyx_v_ndim); - /* "View.MemoryView":1315 + /* "View.MemoryView":1317 * if slice_is_contig(src, 'C', ndim): * direct_copy = slice_is_contig(dst, 'C', ndim) * elif slice_is_contig(src, 'F', ndim): # <<<<<<<<<<<<<< @@ -20599,7 +20662,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ } __pyx_L12:; - /* "View.MemoryView":1318 + /* "View.MemoryView":1320 * direct_copy = slice_is_contig(dst, 'F', ndim) * * if direct_copy: # <<<<<<<<<<<<<< @@ -20609,7 +20672,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_2 = (__pyx_v_direct_copy != 0); if (__pyx_t_2) { - /* "View.MemoryView":1320 + /* "View.MemoryView":1322 * if direct_copy: * * refcount_copying(&dst, dtype_is_object, ndim, False) # <<<<<<<<<<<<<< @@ -20618,7 +20681,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); - /* "View.MemoryView":1321 + /* "View.MemoryView":1323 * * refcount_copying(&dst, dtype_is_object, ndim, False) * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) # <<<<<<<<<<<<<< @@ -20627,7 +20690,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ (void)(memcpy(__pyx_v_dst.data, __pyx_v_src.data, __pyx_memoryview_slice_get_size((&__pyx_v_src), __pyx_v_ndim))); - /* "View.MemoryView":1322 + /* "View.MemoryView":1324 * refcount_copying(&dst, dtype_is_object, ndim, False) * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) * refcount_copying(&dst, dtype_is_object, ndim, True) # <<<<<<<<<<<<<< @@ -20636,7 +20699,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); - /* "View.MemoryView":1323 + /* "View.MemoryView":1325 * memcpy(dst.data, src.data, slice_get_size(&src, ndim)) * refcount_copying(&dst, dtype_is_object, ndim, True) * free(tmpdata) # <<<<<<<<<<<<<< @@ -20645,7 +20708,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ free(__pyx_v_tmpdata); - /* "View.MemoryView":1324 + /* "View.MemoryView":1326 * refcount_copying(&dst, dtype_is_object, ndim, True) * free(tmpdata) * return 0 # <<<<<<<<<<<<<< @@ -20655,7 +20718,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_r = 0; goto __pyx_L0; - /* "View.MemoryView":1318 + /* "View.MemoryView":1320 * direct_copy = slice_is_contig(dst, 'F', ndim) * * if direct_copy: # <<<<<<<<<<<<<< @@ -20664,7 +20727,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ } - /* "View.MemoryView":1310 + /* "View.MemoryView":1312 * src = tmp * * if not broadcasting: # <<<<<<<<<<<<<< @@ -20673,7 +20736,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ } - /* "View.MemoryView":1326 + /* "View.MemoryView":1328 * return 0 * * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< @@ -20687,25 +20750,25 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_t_8 = (__pyx_t_2 != 0); if (__pyx_t_8) { - /* "View.MemoryView":1329 + /* "View.MemoryView":1331 * * * transpose_memslice(&src) # <<<<<<<<<<<<<< * transpose_memslice(&dst) * */ - __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_src)); if (unlikely(__pyx_t_5 == ((int)0))) __PYX_ERR(1, 1329, __pyx_L1_error) + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_src)); if (unlikely(__pyx_t_5 == ((int)0))) __PYX_ERR(1, 1331, __pyx_L1_error) - /* "View.MemoryView":1330 + /* "View.MemoryView":1332 * * transpose_memslice(&src) * transpose_memslice(&dst) # <<<<<<<<<<<<<< * * refcount_copying(&dst, dtype_is_object, ndim, False) */ - __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_dst)); if (unlikely(__pyx_t_5 == ((int)0))) __PYX_ERR(1, 1330, __pyx_L1_error) + __pyx_t_5 = __pyx_memslice_transpose((&__pyx_v_dst)); if (unlikely(__pyx_t_5 == ((int)0))) __PYX_ERR(1, 1332, __pyx_L1_error) - /* "View.MemoryView":1326 + /* "View.MemoryView":1328 * return 0 * * if order == 'F' == get_best_order(&dst, ndim): # <<<<<<<<<<<<<< @@ -20714,7 +20777,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ } - /* "View.MemoryView":1332 + /* "View.MemoryView":1334 * transpose_memslice(&dst) * * refcount_copying(&dst, dtype_is_object, ndim, False) # <<<<<<<<<<<<<< @@ -20723,7 +20786,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 0); - /* "View.MemoryView":1333 + /* "View.MemoryView":1335 * * refcount_copying(&dst, dtype_is_object, ndim, False) * copy_strided_to_strided(&src, &dst, ndim, itemsize) # <<<<<<<<<<<<<< @@ -20732,7 +20795,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ copy_strided_to_strided((&__pyx_v_src), (&__pyx_v_dst), __pyx_v_ndim, __pyx_v_itemsize); - /* "View.MemoryView":1334 + /* "View.MemoryView":1336 * refcount_copying(&dst, dtype_is_object, ndim, False) * copy_strided_to_strided(&src, &dst, ndim, itemsize) * refcount_copying(&dst, dtype_is_object, ndim, True) # <<<<<<<<<<<<<< @@ -20741,7 +20804,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ __pyx_memoryview_refcount_copying((&__pyx_v_dst), __pyx_v_dtype_is_object, __pyx_v_ndim, 1); - /* "View.MemoryView":1336 + /* "View.MemoryView":1338 * refcount_copying(&dst, dtype_is_object, ndim, True) * * free(tmpdata) # <<<<<<<<<<<<<< @@ -20750,7 +20813,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ */ free(__pyx_v_tmpdata); - /* "View.MemoryView":1337 + /* "View.MemoryView":1339 * * free(tmpdata) * return 0 # <<<<<<<<<<<<<< @@ -20760,7 +20823,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ __pyx_r = 0; goto __pyx_L0; - /* "View.MemoryView":1268 + /* "View.MemoryView":1270 * * @cname('__pyx_memoryview_copy_contents') * cdef int memoryview_copy_contents(__Pyx_memviewslice src, # <<<<<<<<<<<<<< @@ -20784,7 +20847,7 @@ static int __pyx_memoryview_copy_contents(__Pyx_memviewslice __pyx_v_src, __Pyx_ return __pyx_r; } -/* "View.MemoryView":1340 +/* "View.MemoryView":1342 * * @cname('__pyx_memoryview_broadcast_leading') * cdef void broadcast_leading(__Pyx_memviewslice *mslice, # <<<<<<<<<<<<<< @@ -20799,7 +20862,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic int __pyx_t_2; int __pyx_t_3; - /* "View.MemoryView":1344 + /* "View.MemoryView":1346 * int ndim_other) nogil: * cdef int i * cdef int offset = ndim_other - ndim # <<<<<<<<<<<<<< @@ -20808,7 +20871,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic */ __pyx_v_offset = (__pyx_v_ndim_other - __pyx_v_ndim); - /* "View.MemoryView":1346 + /* "View.MemoryView":1348 * cdef int offset = ndim_other - ndim * * for i in range(ndim - 1, -1, -1): # <<<<<<<<<<<<<< @@ -20818,7 +20881,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic for (__pyx_t_1 = (__pyx_v_ndim - 1); __pyx_t_1 > -1; __pyx_t_1-=1) { __pyx_v_i = __pyx_t_1; - /* "View.MemoryView":1347 + /* "View.MemoryView":1349 * * for i in range(ndim - 1, -1, -1): * mslice.shape[i + offset] = mslice.shape[i] # <<<<<<<<<<<<<< @@ -20827,7 +20890,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic */ (__pyx_v_mslice->shape[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->shape[__pyx_v_i]); - /* "View.MemoryView":1348 + /* "View.MemoryView":1350 * for i in range(ndim - 1, -1, -1): * mslice.shape[i + offset] = mslice.shape[i] * mslice.strides[i + offset] = mslice.strides[i] # <<<<<<<<<<<<<< @@ -20836,7 +20899,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic */ (__pyx_v_mslice->strides[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->strides[__pyx_v_i]); - /* "View.MemoryView":1349 + /* "View.MemoryView":1351 * mslice.shape[i + offset] = mslice.shape[i] * mslice.strides[i + offset] = mslice.strides[i] * mslice.suboffsets[i + offset] = mslice.suboffsets[i] # <<<<<<<<<<<<<< @@ -20846,7 +20909,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic (__pyx_v_mslice->suboffsets[(__pyx_v_i + __pyx_v_offset)]) = (__pyx_v_mslice->suboffsets[__pyx_v_i]); } - /* "View.MemoryView":1351 + /* "View.MemoryView":1353 * mslice.suboffsets[i + offset] = mslice.suboffsets[i] * * for i in range(offset): # <<<<<<<<<<<<<< @@ -20858,7 +20921,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "View.MemoryView":1352 + /* "View.MemoryView":1354 * * for i in range(offset): * mslice.shape[i] = 1 # <<<<<<<<<<<<<< @@ -20867,7 +20930,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic */ (__pyx_v_mslice->shape[__pyx_v_i]) = 1; - /* "View.MemoryView":1353 + /* "View.MemoryView":1355 * for i in range(offset): * mslice.shape[i] = 1 * mslice.strides[i] = mslice.strides[0] # <<<<<<<<<<<<<< @@ -20876,7 +20939,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic */ (__pyx_v_mslice->strides[__pyx_v_i]) = (__pyx_v_mslice->strides[0]); - /* "View.MemoryView":1354 + /* "View.MemoryView":1356 * mslice.shape[i] = 1 * mslice.strides[i] = mslice.strides[0] * mslice.suboffsets[i] = -1 # <<<<<<<<<<<<<< @@ -20886,7 +20949,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic (__pyx_v_mslice->suboffsets[__pyx_v_i]) = -1L; } - /* "View.MemoryView":1340 + /* "View.MemoryView":1342 * * @cname('__pyx_memoryview_broadcast_leading') * cdef void broadcast_leading(__Pyx_memviewslice *mslice, # <<<<<<<<<<<<<< @@ -20897,7 +20960,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic /* function exit code */ } -/* "View.MemoryView":1362 +/* "View.MemoryView":1364 * * @cname('__pyx_memoryview_refcount_copying') * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, # <<<<<<<<<<<<<< @@ -20908,7 +20971,7 @@ static void __pyx_memoryview_broadcast_leading(__Pyx_memviewslice *__pyx_v_mslic static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_dtype_is_object, int __pyx_v_ndim, int __pyx_v_inc) { int __pyx_t_1; - /* "View.MemoryView":1366 + /* "View.MemoryView":1368 * * * if dtype_is_object: # <<<<<<<<<<<<<< @@ -20918,7 +20981,7 @@ static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *__pyx_v_dst, i __pyx_t_1 = (__pyx_v_dtype_is_object != 0); if (__pyx_t_1) { - /* "View.MemoryView":1367 + /* "View.MemoryView":1369 * * if dtype_is_object: * refcount_objects_in_slice_with_gil(dst.data, dst.shape, # <<<<<<<<<<<<<< @@ -20927,7 +20990,7 @@ static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *__pyx_v_dst, i */ __pyx_memoryview_refcount_objects_in_slice_with_gil(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_inc); - /* "View.MemoryView":1366 + /* "View.MemoryView":1368 * * * if dtype_is_object: # <<<<<<<<<<<<<< @@ -20936,7 +20999,7 @@ static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *__pyx_v_dst, i */ } - /* "View.MemoryView":1362 + /* "View.MemoryView":1364 * * @cname('__pyx_memoryview_refcount_copying') * cdef void refcount_copying(__Pyx_memviewslice *dst, bint dtype_is_object, # <<<<<<<<<<<<<< @@ -20947,7 +21010,7 @@ static void __pyx_memoryview_refcount_copying(__Pyx_memviewslice *__pyx_v_dst, i /* function exit code */ } -/* "View.MemoryView":1371 +/* "View.MemoryView":1373 * * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, # <<<<<<<<<<<<<< @@ -20962,7 +21025,7 @@ static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *__pyx_v_da #endif __Pyx_RefNannySetupContext("refcount_objects_in_slice_with_gil", 0); - /* "View.MemoryView":1374 + /* "View.MemoryView":1376 * Py_ssize_t *strides, int ndim, * bint inc) with gil: * refcount_objects_in_slice(data, shape, strides, ndim, inc) # <<<<<<<<<<<<<< @@ -20971,7 +21034,7 @@ static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *__pyx_v_da */ __pyx_memoryview_refcount_objects_in_slice(__pyx_v_data, __pyx_v_shape, __pyx_v_strides, __pyx_v_ndim, __pyx_v_inc); - /* "View.MemoryView":1371 + /* "View.MemoryView":1373 * * @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') * cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, # <<<<<<<<<<<<<< @@ -20986,7 +21049,7 @@ static void __pyx_memoryview_refcount_objects_in_slice_with_gil(char *__pyx_v_da #endif } -/* "View.MemoryView":1377 +/* "View.MemoryView":1379 * * @cname('__pyx_memoryview_refcount_objects_in_slice') * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, # <<<<<<<<<<<<<< @@ -21003,7 +21066,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss int __pyx_t_4; __Pyx_RefNannySetupContext("refcount_objects_in_slice", 0); - /* "View.MemoryView":1381 + /* "View.MemoryView":1383 * cdef Py_ssize_t i * * for i in range(shape[0]): # <<<<<<<<<<<<<< @@ -21015,7 +21078,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "View.MemoryView":1382 + /* "View.MemoryView":1384 * * for i in range(shape[0]): * if ndim == 1: # <<<<<<<<<<<<<< @@ -21025,7 +21088,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss __pyx_t_4 = ((__pyx_v_ndim == 1) != 0); if (__pyx_t_4) { - /* "View.MemoryView":1383 + /* "View.MemoryView":1385 * for i in range(shape[0]): * if ndim == 1: * if inc: # <<<<<<<<<<<<<< @@ -21035,7 +21098,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss __pyx_t_4 = (__pyx_v_inc != 0); if (__pyx_t_4) { - /* "View.MemoryView":1384 + /* "View.MemoryView":1386 * if ndim == 1: * if inc: * Py_INCREF(( data)[0]) # <<<<<<<<<<<<<< @@ -21044,7 +21107,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss */ Py_INCREF((((PyObject **)__pyx_v_data)[0])); - /* "View.MemoryView":1383 + /* "View.MemoryView":1385 * for i in range(shape[0]): * if ndim == 1: * if inc: # <<<<<<<<<<<<<< @@ -21054,7 +21117,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss goto __pyx_L6; } - /* "View.MemoryView":1386 + /* "View.MemoryView":1388 * Py_INCREF(( data)[0]) * else: * Py_DECREF(( data)[0]) # <<<<<<<<<<<<<< @@ -21066,7 +21129,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss } __pyx_L6:; - /* "View.MemoryView":1382 + /* "View.MemoryView":1384 * * for i in range(shape[0]): * if ndim == 1: # <<<<<<<<<<<<<< @@ -21076,7 +21139,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss goto __pyx_L5; } - /* "View.MemoryView":1388 + /* "View.MemoryView":1390 * Py_DECREF(( data)[0]) * else: * refcount_objects_in_slice(data, shape + 1, strides + 1, # <<<<<<<<<<<<<< @@ -21085,7 +21148,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss */ /*else*/ { - /* "View.MemoryView":1389 + /* "View.MemoryView":1391 * else: * refcount_objects_in_slice(data, shape + 1, strides + 1, * ndim - 1, inc) # <<<<<<<<<<<<<< @@ -21096,7 +21159,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss } __pyx_L5:; - /* "View.MemoryView":1391 + /* "View.MemoryView":1393 * ndim - 1, inc) * * data += strides[0] # <<<<<<<<<<<<<< @@ -21106,7 +21169,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss __pyx_v_data = (__pyx_v_data + (__pyx_v_strides[0])); } - /* "View.MemoryView":1377 + /* "View.MemoryView":1379 * * @cname('__pyx_memoryview_refcount_objects_in_slice') * cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, # <<<<<<<<<<<<<< @@ -21118,7 +21181,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss __Pyx_RefNannyFinishContext(); } -/* "View.MemoryView":1397 +/* "View.MemoryView":1399 * * @cname('__pyx_memoryview_slice_assign_scalar') * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, # <<<<<<<<<<<<<< @@ -21128,7 +21191,7 @@ static void __pyx_memoryview_refcount_objects_in_slice(char *__pyx_v_data, Py_ss static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *__pyx_v_dst, int __pyx_v_ndim, size_t __pyx_v_itemsize, void *__pyx_v_item, int __pyx_v_dtype_is_object) { - /* "View.MemoryView":1400 + /* "View.MemoryView":1402 * size_t itemsize, void *item, * bint dtype_is_object) nogil: * refcount_copying(dst, dtype_is_object, ndim, False) # <<<<<<<<<<<<<< @@ -21137,7 +21200,7 @@ static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *__pyx_v_dst */ __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 0); - /* "View.MemoryView":1401 + /* "View.MemoryView":1403 * bint dtype_is_object) nogil: * refcount_copying(dst, dtype_is_object, ndim, False) * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, # <<<<<<<<<<<<<< @@ -21146,7 +21209,7 @@ static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *__pyx_v_dst */ __pyx_memoryview__slice_assign_scalar(__pyx_v_dst->data, __pyx_v_dst->shape, __pyx_v_dst->strides, __pyx_v_ndim, __pyx_v_itemsize, __pyx_v_item); - /* "View.MemoryView":1403 + /* "View.MemoryView":1405 * _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, * itemsize, item) * refcount_copying(dst, dtype_is_object, ndim, True) # <<<<<<<<<<<<<< @@ -21155,7 +21218,7 @@ static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *__pyx_v_dst */ __pyx_memoryview_refcount_copying(__pyx_v_dst, __pyx_v_dtype_is_object, __pyx_v_ndim, 1); - /* "View.MemoryView":1397 + /* "View.MemoryView":1399 * * @cname('__pyx_memoryview_slice_assign_scalar') * cdef void slice_assign_scalar(__Pyx_memviewslice *dst, int ndim, # <<<<<<<<<<<<<< @@ -21166,7 +21229,7 @@ static void __pyx_memoryview_slice_assign_scalar(__Pyx_memviewslice *__pyx_v_dst /* function exit code */ } -/* "View.MemoryView":1407 +/* "View.MemoryView":1409 * * @cname('__pyx_memoryview__slice_assign_scalar') * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, # <<<<<<<<<<<<<< @@ -21183,7 +21246,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t Py_ssize_t __pyx_t_3; Py_ssize_t __pyx_t_4; - /* "View.MemoryView":1411 + /* "View.MemoryView":1413 * size_t itemsize, void *item) nogil: * cdef Py_ssize_t i * cdef Py_ssize_t stride = strides[0] # <<<<<<<<<<<<<< @@ -21192,7 +21255,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t */ __pyx_v_stride = (__pyx_v_strides[0]); - /* "View.MemoryView":1412 + /* "View.MemoryView":1414 * cdef Py_ssize_t i * cdef Py_ssize_t stride = strides[0] * cdef Py_ssize_t extent = shape[0] # <<<<<<<<<<<<<< @@ -21201,7 +21264,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t */ __pyx_v_extent = (__pyx_v_shape[0]); - /* "View.MemoryView":1414 + /* "View.MemoryView":1416 * cdef Py_ssize_t extent = shape[0] * * if ndim == 1: # <<<<<<<<<<<<<< @@ -21211,7 +21274,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t __pyx_t_1 = ((__pyx_v_ndim == 1) != 0); if (__pyx_t_1) { - /* "View.MemoryView":1415 + /* "View.MemoryView":1417 * * if ndim == 1: * for i in range(extent): # <<<<<<<<<<<<<< @@ -21223,7 +21286,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { __pyx_v_i = __pyx_t_4; - /* "View.MemoryView":1416 + /* "View.MemoryView":1418 * if ndim == 1: * for i in range(extent): * memcpy(data, item, itemsize) # <<<<<<<<<<<<<< @@ -21232,7 +21295,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t */ (void)(memcpy(__pyx_v_data, __pyx_v_item, __pyx_v_itemsize)); - /* "View.MemoryView":1417 + /* "View.MemoryView":1419 * for i in range(extent): * memcpy(data, item, itemsize) * data += stride # <<<<<<<<<<<<<< @@ -21242,7 +21305,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t __pyx_v_data = (__pyx_v_data + __pyx_v_stride); } - /* "View.MemoryView":1414 + /* "View.MemoryView":1416 * cdef Py_ssize_t extent = shape[0] * * if ndim == 1: # <<<<<<<<<<<<<< @@ -21252,7 +21315,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t goto __pyx_L3; } - /* "View.MemoryView":1419 + /* "View.MemoryView":1421 * data += stride * else: * for i in range(extent): # <<<<<<<<<<<<<< @@ -21265,7 +21328,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { __pyx_v_i = __pyx_t_4; - /* "View.MemoryView":1420 + /* "View.MemoryView":1422 * else: * for i in range(extent): * _slice_assign_scalar(data, shape + 1, strides + 1, # <<<<<<<<<<<<<< @@ -21274,7 +21337,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t */ __pyx_memoryview__slice_assign_scalar(__pyx_v_data, (__pyx_v_shape + 1), (__pyx_v_strides + 1), (__pyx_v_ndim - 1), __pyx_v_itemsize, __pyx_v_item); - /* "View.MemoryView":1422 + /* "View.MemoryView":1424 * _slice_assign_scalar(data, shape + 1, strides + 1, * ndim - 1, itemsize, item) * data += stride # <<<<<<<<<<<<<< @@ -21286,7 +21349,7 @@ static void __pyx_memoryview__slice_assign_scalar(char *__pyx_v_data, Py_ssize_t } __pyx_L3:; - /* "View.MemoryView":1407 + /* "View.MemoryView":1409 * * @cname('__pyx_memoryview__slice_assign_scalar') * cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, # <<<<<<<<<<<<<< @@ -23798,9 +23861,9 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 194, __pyx_L1_error) __pyx_builtin_KeyError = __Pyx_GetBuiltinName(__pyx_n_s_KeyError); if (!__pyx_builtin_KeyError) __PYX_ERR(0, 308, __pyx_L1_error) __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 435, __pyx_L1_error) - __pyx_builtin_Ellipsis = __Pyx_GetBuiltinName(__pyx_n_s_Ellipsis); if (!__pyx_builtin_Ellipsis) __PYX_ERR(1, 404, __pyx_L1_error) - __pyx_builtin_id = __Pyx_GetBuiltinName(__pyx_n_s_id); if (!__pyx_builtin_id) __PYX_ERR(1, 613, __pyx_L1_error) - __pyx_builtin_IndexError = __Pyx_GetBuiltinName(__pyx_n_s_IndexError); if (!__pyx_builtin_IndexError) __PYX_ERR(1, 832, __pyx_L1_error) + __pyx_builtin_Ellipsis = __Pyx_GetBuiltinName(__pyx_n_s_Ellipsis); if (!__pyx_builtin_Ellipsis) __PYX_ERR(1, 406, __pyx_L1_error) + __pyx_builtin_id = __Pyx_GetBuiltinName(__pyx_n_s_id); if (!__pyx_builtin_id) __PYX_ERR(1, 615, __pyx_L1_error) + __pyx_builtin_IndexError = __Pyx_GetBuiltinName(__pyx_n_s_IndexError); if (!__pyx_builtin_IndexError) __PYX_ERR(1, 834, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -23950,58 +24013,58 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__15); __Pyx_GIVEREF(__pyx_tuple__15); - /* "View.MemoryView":133 + /* "View.MemoryView":134 * * if not self.ndim: * raise ValueError("Empty shape tuple for cython.array") # <<<<<<<<<<<<<< * * if itemsize <= 0: */ - __pyx_tuple__16 = PyTuple_Pack(1, __pyx_kp_s_Empty_shape_tuple_for_cython_arr); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 133, __pyx_L1_error) + __pyx_tuple__16 = PyTuple_Pack(1, __pyx_kp_s_Empty_shape_tuple_for_cython_arr); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 134, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__16); __Pyx_GIVEREF(__pyx_tuple__16); - /* "View.MemoryView":136 + /* "View.MemoryView":137 * * if itemsize <= 0: * raise ValueError("itemsize <= 0 for cython.array") # <<<<<<<<<<<<<< * * if not isinstance(format, bytes): */ - __pyx_tuple__17 = PyTuple_Pack(1, __pyx_kp_s_itemsize_0_for_cython_array); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 136, __pyx_L1_error) + __pyx_tuple__17 = PyTuple_Pack(1, __pyx_kp_s_itemsize_0_for_cython_array); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 137, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__17); __Pyx_GIVEREF(__pyx_tuple__17); - /* "View.MemoryView":148 + /* "View.MemoryView":149 * * if not self._shape: * raise MemoryError("unable to allocate shape and strides.") # <<<<<<<<<<<<<< * * */ - __pyx_tuple__18 = PyTuple_Pack(1, __pyx_kp_s_unable_to_allocate_shape_and_str); if (unlikely(!__pyx_tuple__18)) __PYX_ERR(1, 148, __pyx_L1_error) + __pyx_tuple__18 = PyTuple_Pack(1, __pyx_kp_s_unable_to_allocate_shape_and_str); if (unlikely(!__pyx_tuple__18)) __PYX_ERR(1, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__18); __Pyx_GIVEREF(__pyx_tuple__18); - /* "View.MemoryView":176 + /* "View.MemoryView":177 * self.data = malloc(self.len) * if not self.data: * raise MemoryError("unable to allocate array data.") # <<<<<<<<<<<<<< * * if self.dtype_is_object: */ - __pyx_tuple__19 = PyTuple_Pack(1, __pyx_kp_s_unable_to_allocate_array_data); if (unlikely(!__pyx_tuple__19)) __PYX_ERR(1, 176, __pyx_L1_error) + __pyx_tuple__19 = PyTuple_Pack(1, __pyx_kp_s_unable_to_allocate_array_data); if (unlikely(!__pyx_tuple__19)) __PYX_ERR(1, 177, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__19); __Pyx_GIVEREF(__pyx_tuple__19); - /* "View.MemoryView":192 + /* "View.MemoryView":193 * bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS * if not (flags & bufmode): * raise ValueError("Can only create a buffer that is contiguous in memory.") # <<<<<<<<<<<<<< * info.buf = self.data * info.len = self.len */ - __pyx_tuple__20 = PyTuple_Pack(1, __pyx_kp_s_Can_only_create_a_buffer_that_is); if (unlikely(!__pyx_tuple__20)) __PYX_ERR(1, 192, __pyx_L1_error) + __pyx_tuple__20 = PyTuple_Pack(1, __pyx_kp_s_Can_only_create_a_buffer_that_is); if (unlikely(!__pyx_tuple__20)) __PYX_ERR(1, 193, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__20); __Pyx_GIVEREF(__pyx_tuple__20); @@ -24024,58 +24087,58 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__22); __Pyx_GIVEREF(__pyx_tuple__22); - /* "View.MemoryView":418 + /* "View.MemoryView":420 * def __setitem__(memoryview self, object index, object value): * if self.view.readonly: * raise TypeError("Cannot assign to read-only memoryview") # <<<<<<<<<<<<<< * * have_slices, index = _unellipsify(index, self.view.ndim) */ - __pyx_tuple__23 = PyTuple_Pack(1, __pyx_kp_s_Cannot_assign_to_read_only_memor); if (unlikely(!__pyx_tuple__23)) __PYX_ERR(1, 418, __pyx_L1_error) + __pyx_tuple__23 = PyTuple_Pack(1, __pyx_kp_s_Cannot_assign_to_read_only_memor); if (unlikely(!__pyx_tuple__23)) __PYX_ERR(1, 420, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__23); __Pyx_GIVEREF(__pyx_tuple__23); - /* "View.MemoryView":495 + /* "View.MemoryView":497 * result = struct.unpack(self.view.format, bytesitem) * except struct.error: * raise ValueError("Unable to convert item to object") # <<<<<<<<<<<<<< * else: * if len(self.view.format) == 1: */ - __pyx_tuple__24 = PyTuple_Pack(1, __pyx_kp_s_Unable_to_convert_item_to_object); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(1, 495, __pyx_L1_error) + __pyx_tuple__24 = PyTuple_Pack(1, __pyx_kp_s_Unable_to_convert_item_to_object); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(1, 497, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__24); __Pyx_GIVEREF(__pyx_tuple__24); - /* "View.MemoryView":520 + /* "View.MemoryView":522 * def __getbuffer__(self, Py_buffer *info, int flags): * if flags & PyBUF_WRITABLE and self.view.readonly: * raise ValueError("Cannot create writable memory view from read-only memoryview") # <<<<<<<<<<<<<< * * if flags & PyBUF_ND: */ - __pyx_tuple__25 = PyTuple_Pack(1, __pyx_kp_s_Cannot_create_writable_memory_vi); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(1, 520, __pyx_L1_error) + __pyx_tuple__25 = PyTuple_Pack(1, __pyx_kp_s_Cannot_create_writable_memory_vi); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(1, 522, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__25); __Pyx_GIVEREF(__pyx_tuple__25); - /* "View.MemoryView":570 + /* "View.MemoryView":572 * if self.view.strides == NULL: * * raise ValueError("Buffer view does not expose strides") # <<<<<<<<<<<<<< * * return tuple([stride for stride in self.view.strides[:self.view.ndim]]) */ - __pyx_tuple__26 = PyTuple_Pack(1, __pyx_kp_s_Buffer_view_does_not_expose_stri); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(1, 570, __pyx_L1_error) + __pyx_tuple__26 = PyTuple_Pack(1, __pyx_kp_s_Buffer_view_does_not_expose_stri); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(1, 572, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__26); __Pyx_GIVEREF(__pyx_tuple__26); - /* "View.MemoryView":577 + /* "View.MemoryView":579 * def suboffsets(self): * if self.view.suboffsets == NULL: * return (-1,) * self.view.ndim # <<<<<<<<<<<<<< * * return tuple([suboffset for suboffset in self.view.suboffsets[:self.view.ndim]]) */ - __pyx_tuple__27 = PyTuple_New(1); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(1, 577, __pyx_L1_error) + __pyx_tuple__27 = PyTuple_New(1); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(1, 579, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__27); __Pyx_INCREF(__pyx_int_neg_1); __Pyx_GIVEREF(__pyx_int_neg_1); @@ -24101,25 +24164,25 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__29); __Pyx_GIVEREF(__pyx_tuple__29); - /* "View.MemoryView":682 + /* "View.MemoryView":684 * if item is Ellipsis: * if not seen_ellipsis: * result.extend([slice(None)] * (ndim - len(tup) + 1)) # <<<<<<<<<<<<<< * seen_ellipsis = True * else: */ - __pyx_slice__30 = PySlice_New(Py_None, Py_None, Py_None); if (unlikely(!__pyx_slice__30)) __PYX_ERR(1, 682, __pyx_L1_error) + __pyx_slice__30 = PySlice_New(Py_None, Py_None, Py_None); if (unlikely(!__pyx_slice__30)) __PYX_ERR(1, 684, __pyx_L1_error) __Pyx_GOTREF(__pyx_slice__30); __Pyx_GIVEREF(__pyx_slice__30); - /* "View.MemoryView":703 + /* "View.MemoryView":705 * for suboffset in suboffsets[:ndim]: * if suboffset >= 0: * raise ValueError("Indirect dimensions not supported") # <<<<<<<<<<<<<< * * */ - __pyx_tuple__31 = PyTuple_Pack(1, __pyx_kp_s_Indirect_dimensions_not_supporte); if (unlikely(!__pyx_tuple__31)) __PYX_ERR(1, 703, __pyx_L1_error) + __pyx_tuple__31 = PyTuple_Pack(1, __pyx_kp_s_Indirect_dimensions_not_supporte); if (unlikely(!__pyx_tuple__31)) __PYX_ERR(1, 705, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__31); __Pyx_GIVEREF(__pyx_tuple__31); @@ -24145,58 +24208,58 @@ static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__34); __Pyx_GIVEREF(__pyx_tuple__34); - /* "View.MemoryView":286 + /* "View.MemoryView":287 * return self.name * * cdef generic = Enum("") # <<<<<<<<<<<<<< * cdef strided = Enum("") # default * cdef indirect = Enum("") */ - __pyx_tuple__36 = PyTuple_Pack(1, __pyx_kp_s_strided_and_direct_or_indirect); if (unlikely(!__pyx_tuple__36)) __PYX_ERR(1, 286, __pyx_L1_error) + __pyx_tuple__36 = PyTuple_Pack(1, __pyx_kp_s_strided_and_direct_or_indirect); if (unlikely(!__pyx_tuple__36)) __PYX_ERR(1, 287, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__36); __Pyx_GIVEREF(__pyx_tuple__36); - /* "View.MemoryView":287 + /* "View.MemoryView":288 * * cdef generic = Enum("") * cdef strided = Enum("") # default # <<<<<<<<<<<<<< * cdef indirect = Enum("") * */ - __pyx_tuple__37 = PyTuple_Pack(1, __pyx_kp_s_strided_and_direct); if (unlikely(!__pyx_tuple__37)) __PYX_ERR(1, 287, __pyx_L1_error) + __pyx_tuple__37 = PyTuple_Pack(1, __pyx_kp_s_strided_and_direct); if (unlikely(!__pyx_tuple__37)) __PYX_ERR(1, 288, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__37); __Pyx_GIVEREF(__pyx_tuple__37); - /* "View.MemoryView":288 + /* "View.MemoryView":289 * cdef generic = Enum("") * cdef strided = Enum("") # default * cdef indirect = Enum("") # <<<<<<<<<<<<<< * * */ - __pyx_tuple__38 = PyTuple_Pack(1, __pyx_kp_s_strided_and_indirect); if (unlikely(!__pyx_tuple__38)) __PYX_ERR(1, 288, __pyx_L1_error) + __pyx_tuple__38 = PyTuple_Pack(1, __pyx_kp_s_strided_and_indirect); if (unlikely(!__pyx_tuple__38)) __PYX_ERR(1, 289, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__38); __Pyx_GIVEREF(__pyx_tuple__38); - /* "View.MemoryView":291 + /* "View.MemoryView":292 * * * cdef contiguous = Enum("") # <<<<<<<<<<<<<< * cdef indirect_contiguous = Enum("") * */ - __pyx_tuple__39 = PyTuple_Pack(1, __pyx_kp_s_contiguous_and_direct); if (unlikely(!__pyx_tuple__39)) __PYX_ERR(1, 291, __pyx_L1_error) + __pyx_tuple__39 = PyTuple_Pack(1, __pyx_kp_s_contiguous_and_direct); if (unlikely(!__pyx_tuple__39)) __PYX_ERR(1, 292, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__39); __Pyx_GIVEREF(__pyx_tuple__39); - /* "View.MemoryView":292 + /* "View.MemoryView":293 * * cdef contiguous = Enum("") * cdef indirect_contiguous = Enum("") # <<<<<<<<<<<<<< * * */ - __pyx_tuple__40 = PyTuple_Pack(1, __pyx_kp_s_contiguous_and_indirect); if (unlikely(!__pyx_tuple__40)) __PYX_ERR(1, 292, __pyx_L1_error) + __pyx_tuple__40 = PyTuple_Pack(1, __pyx_kp_s_contiguous_and_indirect); if (unlikely(!__pyx_tuple__40)) __PYX_ERR(1, 293, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__40); __Pyx_GIVEREF(__pyx_tuple__40); @@ -24374,21 +24437,21 @@ static int __Pyx_modinit_type_init_code(void) { __pyx_ptype_9csimdjson___pyx_scope_struct_4_get_implementations = &__pyx_type_9csimdjson___pyx_scope_struct_4_get_implementations; __pyx_vtabptr_array = &__pyx_vtable_array; __pyx_vtable_array.get_memview = (PyObject *(*)(struct __pyx_array_obj *))__pyx_array_get_memview; - if (PyType_Ready(&__pyx_type___pyx_array) < 0) __PYX_ERR(1, 105, __pyx_L1_error) + if (PyType_Ready(&__pyx_type___pyx_array) < 0) __PYX_ERR(1, 106, __pyx_L1_error) #if PY_VERSION_HEX < 0x030800B1 __pyx_type___pyx_array.tp_print = 0; #endif - if (__Pyx_SetVtable(__pyx_type___pyx_array.tp_dict, __pyx_vtabptr_array) < 0) __PYX_ERR(1, 105, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type___pyx_array) < 0) __PYX_ERR(1, 105, __pyx_L1_error) + if (__Pyx_SetVtable(__pyx_type___pyx_array.tp_dict, __pyx_vtabptr_array) < 0) __PYX_ERR(1, 106, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type___pyx_array) < 0) __PYX_ERR(1, 106, __pyx_L1_error) __pyx_array_type = &__pyx_type___pyx_array; - if (PyType_Ready(&__pyx_type___pyx_MemviewEnum) < 0) __PYX_ERR(1, 279, __pyx_L1_error) + if (PyType_Ready(&__pyx_type___pyx_MemviewEnum) < 0) __PYX_ERR(1, 280, __pyx_L1_error) #if PY_VERSION_HEX < 0x030800B1 __pyx_type___pyx_MemviewEnum.tp_print = 0; #endif if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type___pyx_MemviewEnum.tp_dictoffset && __pyx_type___pyx_MemviewEnum.tp_getattro == PyObject_GenericGetAttr)) { __pyx_type___pyx_MemviewEnum.tp_getattro = __Pyx_PyObject_GenericGetAttr; } - if (__Pyx_setup_reduce((PyObject*)&__pyx_type___pyx_MemviewEnum) < 0) __PYX_ERR(1, 279, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type___pyx_MemviewEnum) < 0) __PYX_ERR(1, 280, __pyx_L1_error) __pyx_MemviewEnum_type = &__pyx_type___pyx_MemviewEnum; __pyx_vtabptr_memoryview = &__pyx_vtable_memoryview; __pyx_vtable_memoryview.get_item_pointer = (char *(*)(struct __pyx_memoryview_obj *, PyObject *))__pyx_memoryview_get_item_pointer; @@ -24398,30 +24461,30 @@ static int __Pyx_modinit_type_init_code(void) { __pyx_vtable_memoryview.setitem_indexed = (PyObject *(*)(struct __pyx_memoryview_obj *, PyObject *, PyObject *))__pyx_memoryview_setitem_indexed; __pyx_vtable_memoryview.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryview_convert_item_to_object; __pyx_vtable_memoryview.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryview_assign_item_from_object; - if (PyType_Ready(&__pyx_type___pyx_memoryview) < 0) __PYX_ERR(1, 330, __pyx_L1_error) + if (PyType_Ready(&__pyx_type___pyx_memoryview) < 0) __PYX_ERR(1, 331, __pyx_L1_error) #if PY_VERSION_HEX < 0x030800B1 __pyx_type___pyx_memoryview.tp_print = 0; #endif if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type___pyx_memoryview.tp_dictoffset && __pyx_type___pyx_memoryview.tp_getattro == PyObject_GenericGetAttr)) { __pyx_type___pyx_memoryview.tp_getattro = __Pyx_PyObject_GenericGetAttr; } - if (__Pyx_SetVtable(__pyx_type___pyx_memoryview.tp_dict, __pyx_vtabptr_memoryview) < 0) __PYX_ERR(1, 330, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type___pyx_memoryview) < 0) __PYX_ERR(1, 330, __pyx_L1_error) + if (__Pyx_SetVtable(__pyx_type___pyx_memoryview.tp_dict, __pyx_vtabptr_memoryview) < 0) __PYX_ERR(1, 331, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type___pyx_memoryview) < 0) __PYX_ERR(1, 331, __pyx_L1_error) __pyx_memoryview_type = &__pyx_type___pyx_memoryview; __pyx_vtabptr__memoryviewslice = &__pyx_vtable__memoryviewslice; __pyx_vtable__memoryviewslice.__pyx_base = *__pyx_vtabptr_memoryview; __pyx_vtable__memoryviewslice.__pyx_base.convert_item_to_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *))__pyx_memoryviewslice_convert_item_to_object; __pyx_vtable__memoryviewslice.__pyx_base.assign_item_from_object = (PyObject *(*)(struct __pyx_memoryview_obj *, char *, PyObject *))__pyx_memoryviewslice_assign_item_from_object; __pyx_type___pyx_memoryviewslice.tp_base = __pyx_memoryview_type; - if (PyType_Ready(&__pyx_type___pyx_memoryviewslice) < 0) __PYX_ERR(1, 965, __pyx_L1_error) + if (PyType_Ready(&__pyx_type___pyx_memoryviewslice) < 0) __PYX_ERR(1, 967, __pyx_L1_error) #if PY_VERSION_HEX < 0x030800B1 __pyx_type___pyx_memoryviewslice.tp_print = 0; #endif if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type___pyx_memoryviewslice.tp_dictoffset && __pyx_type___pyx_memoryviewslice.tp_getattro == PyObject_GenericGetAttr)) { __pyx_type___pyx_memoryviewslice.tp_getattro = __Pyx_PyObject_GenericGetAttr; } - if (__Pyx_SetVtable(__pyx_type___pyx_memoryviewslice.tp_dict, __pyx_vtabptr__memoryviewslice) < 0) __PYX_ERR(1, 965, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type___pyx_memoryviewslice) < 0) __PYX_ERR(1, 965, __pyx_L1_error) + if (__Pyx_SetVtable(__pyx_type___pyx_memoryviewslice.tp_dict, __pyx_vtabptr__memoryviewslice) < 0) __PYX_ERR(1, 967, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type___pyx_memoryviewslice) < 0) __PYX_ERR(1, 967, __pyx_L1_error) __pyx_memoryviewslice_type = &__pyx_type___pyx_memoryviewslice; __Pyx_RefNannyFinishContext(); return 0; @@ -24812,90 +24875,90 @@ if (!__Pyx_RefNanny) { if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_4) < 0) __PYX_ERR(0, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":209 + /* "View.MemoryView":210 * info.obj = self * * __pyx_getbuffer = capsule( &__pyx_array_getbuffer, "getbuffer(obj, view, flags)") # <<<<<<<<<<<<<< * * def __dealloc__(array self): */ - __pyx_t_4 = __pyx_capsule_create(((void *)(&__pyx_array_getbuffer)), ((char *)"getbuffer(obj, view, flags)")); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 209, __pyx_L1_error) + __pyx_t_4 = __pyx_capsule_create(((void *)(&__pyx_array_getbuffer)), ((char *)"getbuffer(obj, view, flags)")); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 210, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - if (PyDict_SetItem((PyObject *)__pyx_array_type->tp_dict, __pyx_n_s_pyx_getbuffer, __pyx_t_4) < 0) __PYX_ERR(1, 209, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_array_type->tp_dict, __pyx_n_s_pyx_getbuffer, __pyx_t_4) < 0) __PYX_ERR(1, 210, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; PyType_Modified(__pyx_array_type); - /* "View.MemoryView":286 + /* "View.MemoryView":287 * return self.name * * cdef generic = Enum("") # <<<<<<<<<<<<<< * cdef strided = Enum("") # default * cdef indirect = Enum("") */ - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__36, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 286, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__36, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 287, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_XGOTREF(generic); __Pyx_DECREF_SET(generic, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":287 + /* "View.MemoryView":288 * * cdef generic = Enum("") * cdef strided = Enum("") # default # <<<<<<<<<<<<<< * cdef indirect = Enum("") * */ - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__37, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 287, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__37, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 288, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_XGOTREF(strided); __Pyx_DECREF_SET(strided, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":288 + /* "View.MemoryView":289 * cdef generic = Enum("") * cdef strided = Enum("") # default * cdef indirect = Enum("") # <<<<<<<<<<<<<< * * */ - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__38, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 288, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__38, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 289, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_XGOTREF(indirect); __Pyx_DECREF_SET(indirect, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":291 + /* "View.MemoryView":292 * * * cdef contiguous = Enum("") # <<<<<<<<<<<<<< * cdef indirect_contiguous = Enum("") * */ - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__39, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 291, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__39, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 292, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_XGOTREF(contiguous); __Pyx_DECREF_SET(contiguous, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":292 + /* "View.MemoryView":293 * * cdef contiguous = Enum("") * cdef indirect_contiguous = Enum("") # <<<<<<<<<<<<<< * * */ - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__40, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 292, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_MemviewEnum_type), __pyx_tuple__40, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 293, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_XGOTREF(indirect_contiguous); __Pyx_DECREF_SET(indirect_contiguous, __pyx_t_4); __Pyx_GIVEREF(__pyx_t_4); __pyx_t_4 = 0; - /* "View.MemoryView":316 + /* "View.MemoryView":317 * * DEF THREAD_LOCKS_PREALLOCATED = 8 * cdef int __pyx_memoryview_thread_locks_used = 0 # <<<<<<<<<<<<<< @@ -24904,7 +24967,7 @@ if (!__Pyx_RefNanny) { */ __pyx_memoryview_thread_locks_used = 0; - /* "View.MemoryView":317 + /* "View.MemoryView":318 * DEF THREAD_LOCKS_PREALLOCATED = 8 * cdef int __pyx_memoryview_thread_locks_used = 0 * cdef PyThread_type_lock[THREAD_LOCKS_PREALLOCATED] __pyx_memoryview_thread_locks = [ # <<<<<<<<<<<<<< @@ -24921,29 +24984,29 @@ if (!__Pyx_RefNanny) { __pyx_t_5[7] = PyThread_allocate_lock(); memcpy(&(__pyx_memoryview_thread_locks[0]), __pyx_t_5, sizeof(__pyx_memoryview_thread_locks[0]) * (8)); - /* "View.MemoryView":549 + /* "View.MemoryView":551 * info.obj = self * * __pyx_getbuffer = capsule( &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)") # <<<<<<<<<<<<<< * * */ - __pyx_t_4 = __pyx_capsule_create(((void *)(&__pyx_memoryview_getbuffer)), ((char *)"getbuffer(obj, view, flags)")); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 549, __pyx_L1_error) + __pyx_t_4 = __pyx_capsule_create(((void *)(&__pyx_memoryview_getbuffer)), ((char *)"getbuffer(obj, view, flags)")); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 551, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - if (PyDict_SetItem((PyObject *)__pyx_memoryview_type->tp_dict, __pyx_n_s_pyx_getbuffer, __pyx_t_4) < 0) __PYX_ERR(1, 549, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_memoryview_type->tp_dict, __pyx_n_s_pyx_getbuffer, __pyx_t_4) < 0) __PYX_ERR(1, 551, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; PyType_Modified(__pyx_memoryview_type); - /* "View.MemoryView":995 + /* "View.MemoryView":997 * return self.from_object * * __pyx_getbuffer = capsule( &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)") # <<<<<<<<<<<<<< * * */ - __pyx_t_4 = __pyx_capsule_create(((void *)(&__pyx_memoryview_getbuffer)), ((char *)"getbuffer(obj, view, flags)")); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 995, __pyx_L1_error) + __pyx_t_4 = __pyx_capsule_create(((void *)(&__pyx_memoryview_getbuffer)), ((char *)"getbuffer(obj, view, flags)")); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 997, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - if (PyDict_SetItem((PyObject *)__pyx_memoryviewslice_type->tp_dict, __pyx_n_s_pyx_getbuffer, __pyx_t_4) < 0) __PYX_ERR(1, 995, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_memoryviewslice_type->tp_dict, __pyx_n_s_pyx_getbuffer, __pyx_t_4) < 0) __PYX_ERR(1, 997, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; PyType_Modified(__pyx_memoryviewslice_type); @@ -25981,7 +26044,7 @@ __Pyx_init_memviewslice(struct __pyx_memoryview_obj *memview, static void __pyx_fatalerror(const char *fmt, ...) Py_NO_RETURN { va_list vargs; char msg[200]; -#ifdef HAVE_STDARG_PROTOTYPES +#if PY_VERSION_HEX >= 0x030A0000 || defined(HAVE_STDARG_PROTOTYPES) va_start(vargs, fmt); #else va_start(vargs); @@ -27163,7 +27226,7 @@ static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, char* chars, padding = PyUnicode_FromOrdinal(padding_char); if (likely(padding) && uoffset > prepend_sign + 1) { PyObject *tmp; - PyObject *repeat = PyInt_FromSize_t(uoffset - prepend_sign); + PyObject *repeat = PyInt_FromSsize_t(uoffset - prepend_sign); if (unlikely(!repeat)) goto done_or_error; tmp = PyNumber_Multiply(padding, repeat); Py_DECREF(repeat); diff --git a/simdjson/simdjson.cpp b/simdjson/simdjson.cpp index c98c4b1..611411a 100644 --- a/simdjson/simdjson.cpp +++ b/simdjson/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2022-06-02 13:56:20 -0400. Do not edit! */ +/* auto-generated on 2022-11-23 10:31:42 -0500. Do not edit! */ /* begin file src/simdjson.cpp */ #include "simdjson.h" @@ -868,9 +868,9 @@ inline char *format_buffer(char *buf, int len, int decimal_exponent, std::memset(buf + k, '0', static_cast(n) - static_cast(k)); // Make it look like a floating-point number (#362, #378) - // buf[n + 0] = '.'; - // buf[n + 1] = '0'; - return buf + (static_cast(n)); + buf[n + 0] = '.'; + buf[n + 1] = '0'; + return buf + (static_cast(n)) + 2; } if (0 < n && n <= max_exp) { @@ -933,10 +933,8 @@ char *to_chars(char *first, const char *last, double value) { { *first++ = '0'; // Make it look like a floating-point number (#362, #378) - if(negative) { - *first++ = '.'; - *first++ = '0'; - } + *first++ = '.'; + *first++ = '0'; return first; } // Compute v = buffer * 10^decimal_exponent. @@ -1589,7 +1587,8 @@ namespace internal { { INSUFFICIENT_PADDING, "simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length. Consider using the simdjson::padded_string class if needed." }, { INCOMPLETE_ARRAY_OR_OBJECT, "JSON document ended early in the middle of an object or array." }, { SCALAR_DOCUMENT_AS_VALUE, "A JSON document made of a scalar (number, Boolean, null or string) is treated as a value. Use get_bool(), get_double(), etc. on the document instead. "}, - { OUT_OF_BOUNDS, "Attempted to access location outside of document."} + { OUT_OF_BOUNDS, "Attempted to access location outside of document."}, + { TRAILING_CONTENT, "Unexpected trailing content in the JSON input."} }; // error_messages[] } // namespace internal @@ -2677,7 +2676,7 @@ class detect_best_supported_implementation_on_first_use final : public implement simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) const noexcept final override { return set_best()->validate_utf8(buf, len); } - simdjson_really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {} + simdjson_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {} private: const implementation *set_best() const noexcept; }; @@ -2793,7 +2792,6 @@ simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, s simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept { return get_active_implementation()->validate_utf8(buf, len); } - const implementation * builtin_implementation() { static const implementation * builtin_impl = get_available_implementations()[SIMDJSON_STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)]; assert(builtin_impl); @@ -2850,17 +2848,17 @@ namespace { using namespace simd; struct json_character_block { - static simdjson_really_inline json_character_block classify(const simd::simd8x64& in); + static simdjson_inline json_character_block classify(const simd::simd8x64& in); - simdjson_really_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_really_inline uint64_t op() const noexcept { return _op; } - simdjson_really_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } + simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } + simdjson_inline uint64_t op() const noexcept { return _op; } + simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } uint64_t _whitespace; uint64_t _op; }; -simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { +simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // Functional programming causes trouble with Visual Studio. // Keeping this version in comments since it is much nicer: // auto v = in.map([&](simd8 chunk) { @@ -2914,15 +2912,15 @@ simdjson_really_inline json_character_block json_character_block::classify(const return { whitespace, op }; } -simdjson_really_inline bool is_ascii(const simd8x64& input) { +simdjson_inline bool is_ascii(const simd8x64& input) { simd8 bits = input.reduce_or(); - return bits.max_val() < 0b10000000u; + return bits.max_val() < 0x80u; } -simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1 >= uint8_t(0b11000000u); - simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); - simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); +simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1 >= uint8_t(0xc0u); + simd8 is_third_byte = prev2 >= uint8_t(0xe0u); + simd8 is_fourth_byte = prev3 >= uint8_t(0xf0u); // Use ^ instead of | for is_*_byte, because ^ is commutative, and the caller is using ^ as well. // This will work fine because we only have to report errors for cases with 0-1 lead bytes. // Multiple lead bytes implies 2 overlapping multibyte characters, and if that happens, there is @@ -2931,9 +2929,9 @@ simdjson_unused simdjson_really_inline simd8 must_be_continuation(const si return is_second_byte ^ is_third_byte ^ is_fourth_byte; } -simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2 >= uint8_t(0b11100000u); - simd8 is_fourth_byte = prev3 >= uint8_t(0b11110000u); +simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2 >= uint8_t(0xe0u); + simd8 is_fourth_byte = prev3 >= uint8_t(0xf0u); return is_third_byte ^ is_fourth_byte; } @@ -2949,7 +2947,7 @@ namespace utf8_validation { using namespace simd; - simdjson_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -3039,7 +3037,7 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - simdjson_really_inline simd8 check_multibyte_lengths(const simd8 input, + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -3052,7 +3050,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - simdjson_really_inline simd8 is_incomplete(const simd8 input) { + simdjson_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ #if SIMDJSON_IMPLEMENTATION_ICELAKE @@ -3064,14 +3062,14 @@ using namespace simd; 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #else static const uint8_t max_array[32] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #endif const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); @@ -3089,7 +3087,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - simdjson_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -3100,13 +3098,21 @@ using namespace simd; // The only problem that can happen at EOF is that a multibyte character is too short // or a byte value too large in the last bytes: check_special_cases only checks for bytes // too large in the first of two bytes. - simdjson_really_inline void check_eof() { + simdjson_inline void check_eof() { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. this->error |= this->prev_incomplete; } - simdjson_really_inline void check_next_input(const simd8x64& input) { +#ifndef SIMDJSON_IF_CONSTEXPR +#if SIMDJSON_CPLUSPLUS17 +#define SIMDJSON_IF_CONSTEXPR if constexpr +#else +#define SIMDJSON_IF_CONSTEXPR if +#endif +#endif + + simdjson_inline void check_next_input(const simd8x64& input) { if(simdjson_likely(is_ascii(input))) { this->error |= this->prev_incomplete; } else { @@ -3115,12 +3121,12 @@ using namespace simd; ||(simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support one, two or four chunks per 64-byte block."); - if(simd8x64::NUM_CHUNKS == 1) { + SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - } if(simd8x64::NUM_CHUNKS == 2) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if(simd8x64::NUM_CHUNKS == 4) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -3131,7 +3137,7 @@ using namespace simd; } } // do not forget to call check_eof! - simdjson_really_inline error_code errors() { + simdjson_inline error_code errors() { return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; } @@ -3159,10 +3165,10 @@ namespace { template struct buf_block_reader { public: - simdjson_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_really_inline size_t block_index(); - simdjson_really_inline bool has_full_block() const; - simdjson_really_inline const uint8_t *full_block() const; + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; /** * Get the last block, padded with spaces. * @@ -3172,8 +3178,8 @@ struct buf_block_reader { * * @return the number of effective characters in the last block. */ - simdjson_really_inline size_t get_remainder(uint8_t *dst) const; - simdjson_really_inline void advance(); + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); private: const uint8_t *buf; const size_t len; @@ -3212,23 +3218,23 @@ simdjson_unused static char * format_mask(uint64_t mask) { } template -simdjson_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} template -simdjson_really_inline size_t buf_block_reader::block_index() { return idx; } +simdjson_inline size_t buf_block_reader::block_index() { return idx; } template -simdjson_really_inline bool buf_block_reader::has_full_block() const { +simdjson_inline bool buf_block_reader::has_full_block() const { return idx < lenminusstep; } template -simdjson_really_inline const uint8_t *buf_block_reader::full_block() const { +simdjson_inline const uint8_t *buf_block_reader::full_block() const { return &buf[idx]; } template -simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. std::memcpy(dst, buf + idx, len - idx); @@ -3236,7 +3242,7 @@ simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t } template -simdjson_really_inline void buf_block_reader::advance() { +simdjson_inline void buf_block_reader::advance() { idx += STEP_SIZE; } @@ -3252,27 +3258,27 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : + simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } + simdjson_inline uint64_t escaped() const { return _escaped; } // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_really_inline uint64_t escape() const { return _backslash & ~_escaped; } + simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } + simdjson_inline uint64_t quote() const { return _quote; } // Start quotes of strings - simdjson_really_inline uint64_t string_start() const { return _quote & _in_string; } + simdjson_inline uint64_t string_start() const { return _quote & _in_string; } // End quotes of strings - simdjson_really_inline uint64_t string_end() const { return _quote & ~_in_string; } + simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } // backslash characters uint64_t _backslash; @@ -3287,14 +3293,14 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); + simdjson_inline json_string_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Intended to be defined by the implementation - simdjson_really_inline uint64_t find_escaped(uint64_t escape); - simdjson_really_inline uint64_t find_escaped_branchless(uint64_t escape); + simdjson_inline uint64_t find_escaped(uint64_t escape); + simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -3329,7 +3335,7 @@ class json_string_scanner { // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -3354,7 +3360,7 @@ simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uin // // Backslash sequences outside of quotes will be detected in stage 2. // -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -3387,7 +3393,7 @@ simdjson_really_inline json_string_block json_string_scanner::next(const simd::s ); } -simdjson_really_inline error_code json_string_scanner::finish() { +simdjson_inline error_code json_string_scanner::finish() { if (prev_in_string) { return UNCLOSED_STRING; } @@ -3425,25 +3431,25 @@ namespace stage1 { struct json_block { public: // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_really_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} /** * The start of structurals. * In simdjson prior to v0.3, these were called the pseudo-structural characters. **/ - simdjson_really_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } /** All JSON whitespace (i.e. not in a string) */ - simdjson_really_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } // Helpers /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } // string and escape characters json_string_block _string; @@ -3458,12 +3464,12 @@ struct json_block { * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". * They may reside inside a string. **/ - simdjson_really_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } /** * The start of non-operator runs, like 123, true and "abc". * It main reside inside a string. **/ - simdjson_really_inline uint64_t potential_scalar_start() const noexcept { + simdjson_inline uint64_t potential_scalar_start() const noexcept { // The term "scalar" refers to anything except structural characters and white space // (so letters, numbers, quotes). // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space @@ -3474,7 +3480,7 @@ struct json_block { * Whether the given character is immediately after a non-operator like 123, true. * The characters following a quote are not included. */ - simdjson_really_inline uint64_t follows_potential_scalar() const noexcept { + simdjson_inline uint64_t follows_potential_scalar() const noexcept { // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a // white space. @@ -3498,10 +3504,10 @@ struct json_block { */ class json_scanner { public: - json_scanner() {} - simdjson_really_inline json_block next(const simd::simd8x64& in); + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Whether the last character of the previous iteration is part of a scalar token @@ -3518,13 +3524,13 @@ class json_scanner { // // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); // -simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { +simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { const uint64_t result = match << 1 | overflow; overflow = match >> 63; return result; } -simdjson_really_inline json_block json_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); // identifies the white-space and the structural characters json_character_block characters = json_character_block::classify(in); @@ -3549,7 +3555,7 @@ simdjson_really_inline json_block json_scanner::next(const simd::simd8x64 - simdjson_really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); + simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -simdjson_really_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { +simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { uint64_t mask = block.whitespace(); dst += in.compress(mask, dst); } -simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { +simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { error_code error = scanner.finish(); if (error) { dst_len = 0; return error; } dst_len = dst - dst_start; @@ -3599,7 +3605,7 @@ simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size } template<> -simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block_buf); simd::simd8x64 in_2(block_buf+64); json_block block_1 = scanner.next(in_1); @@ -3610,7 +3616,7 @@ simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, b } template<> -simdjson_really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block_buf); json_block block_1 = scanner.next(in_1); this->next(block_buf, block_1); @@ -3683,7 +3689,7 @@ namespace { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -3758,7 +3764,7 @@ class bit_indexer { public: uint32_t *tail; - simdjson_really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at @@ -3769,9 +3775,9 @@ class bit_indexer { // If the kernel sets SIMDJSON_CUSTOM_BIT_INDEXER, then it will provide its own // version of the code. #ifdef SIMDJSON_CUSTOM_BIT_INDEXER - simdjson_really_inline void write(uint32_t idx, uint64_t bits); + simdjson_inline void write(uint32_t idx, uint64_t bits); #else - simdjson_really_inline void write(uint32_t idx, uint64_t bits) { + simdjson_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -3880,11 +3886,11 @@ class json_structural_indexer { static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; private: - simdjson_really_inline json_structural_indexer(uint32_t *structural_indexes); + simdjson_inline json_structural_indexer(uint32_t *structural_indexes); template - simdjson_really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); + simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); json_scanner scanner{}; utf8_checker checker{}; @@ -3893,26 +3899,26 @@ class json_structural_indexer { uint64_t unescaped_chars_error = 0; }; -simdjson_really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} +simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} // Skip the last character if it is partial -simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { +simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { if (simdjson_unlikely(len < 3)) { switch (len) { case 2: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 2 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left return len; case 1: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left return len; case 0: return len; } } - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left return len; } @@ -3961,7 +3967,7 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_pa } template<> -simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block); simd::simd8x64 in_2(block+64); json_block block_1 = scanner.next(in_1); @@ -3972,14 +3978,14 @@ simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *bl } template<> -simdjson_really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block); json_block block_1 = scanner.next(in_1); this->next(in_1, block_1, reader.block_index()); reader.advance(); } -simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { +simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -3987,7 +3993,7 @@ simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64 backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + return false; + } + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + return false; + } + + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + return false; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_nullptrptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +} // namespace stringparsing +} // unnamed namespace +} // namespace arm64 +} // namespace simdjson +/* end file src/generic/stage2/stringparsing.h */ /* begin file src/generic/stage2/tape_builder.h */ /* begin file src/generic/stage2/json_iterator.h */ /* begin file src/generic/stage2/logger.h */ @@ -4159,7 +4310,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static simdjson_really_inline char printable_char(char c) { + static simdjson_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -4168,7 +4319,7 @@ namespace logger { } // Print the header and set up log_start - static simdjson_really_inline void log_start() { + static simdjson_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -4177,7 +4328,7 @@ namespace logger { } } - simdjson_unused static simdjson_really_inline void log_string(const char *message) { + simdjson_unused static simdjson_inline void log_string(const char *message) { if (LOG_ENABLED) { printf("%s\n", message); } @@ -4185,7 +4336,7 @@ namespace logger { // Logs a single line from the stage 2 DOM parser template - static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -4263,14 +4414,14 @@ class json_iterator { * - increment_count(iter) - each time a value is found in an array or object. */ template - simdjson_warn_unused simdjson_really_inline error_code walk_document(V &visitor) noexcept; + simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -4279,7 +4430,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *peek() const noexcept; + simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -4287,56 +4438,56 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *advance() noexcept; + simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - simdjson_really_inline size_t remaining_len() const noexcept; + simdjson_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - simdjson_really_inline bool at_eof() const noexcept; + simdjson_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - simdjson_really_inline bool at_beginning() const noexcept; - simdjson_really_inline uint8_t last_structural() const noexcept; + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_value(const char *type) const noexcept; + simdjson_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_start_value(const char *type) const noexcept; + simdjson_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_end_value(const char *type) const noexcept; + simdjson_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_error(const char *error) const noexcept; + simdjson_inline void log_error(const char *error) const noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -4461,52 +4612,52 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_docum } // walk_document() -simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -simdjson_really_inline bool json_iterator::at_eof() const noexcept { +simdjson_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -simdjson_really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -4522,7 +4673,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root } } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -4554,18 +4705,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - simdjson_really_inline void append_s64(int64_t value) noexcept; + simdjson_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - simdjson_really_inline void append_u64(uint64_t value) noexcept; + simdjson_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - simdjson_really_inline void append_double(double value) noexcept; + simdjson_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -4573,24 +4724,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - simdjson_really_inline void skip() noexcept; + simdjson_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - simdjson_really_inline void skip_large_integer() noexcept; + simdjson_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - simdjson_really_inline void skip_double() noexcept; + simdjson_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -4598,50 +4749,50 @@ struct tape_writer { * all 64 bits, such as double and uint64_t. */ template - simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -simdjson_really_inline void tape_writer::append_double(double value) noexcept { +simdjson_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -simdjson_really_inline void tape_writer::skip() noexcept { +simdjson_inline void tape_writer::skip() noexcept { next_tape_loc++; } -simdjson_really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::skip_double() noexcept { +simdjson_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -4658,40 +4809,40 @@ namespace stage2 { struct tape_builder { template - simdjson_warn_unused static simdjson_really_inline error_code parse_document( + simdjson_warn_unused static simdjson_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - simdjson_warn_unused simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -4700,22 +4851,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -4723,18 +4874,18 @@ struct tape_builder { /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_really_inline tape_builder(dom::document &doc) noexcept; + simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_really_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_inline void start_container(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_document( +simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -4743,56 +4894,56 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_docum return iter.walk_document(builder); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value+1, dst); @@ -4804,16 +4955,16 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_strin return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -4835,42 +4986,42 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ return error; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -4879,24 +5030,24 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ // private: -simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -4909,13 +5060,13 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_contain return SUCCESS; } -simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -4941,7 +5092,7 @@ namespace arm64 { namespace { namespace stage1 { -simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { // On ARM, we don't short-circuit this if there are no backslashes, because the branch gives us no // benefit and therefore makes things worse. // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } @@ -4973,6 +5124,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { + return arm64::stringparsing::parse_string(src, dst); +} + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } @@ -5055,7 +5210,7 @@ namespace { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -5129,7 +5284,7 @@ namespace stage1 { class structural_scanner { public: -simdjson_really_inline structural_scanner(dom_parser_implementation &_parser, stage1_mode _partial) +simdjson_inline structural_scanner(dom_parser_implementation &_parser, stage1_mode _partial) : buf{_parser.buf}, next_structural_index{_parser.structural_indexes.get()}, parser{_parser}, @@ -5137,18 +5292,18 @@ simdjson_really_inline structural_scanner(dom_parser_implementation &_parser, st partial{_partial} { } -simdjson_really_inline void add_structural() { +simdjson_inline void add_structural() { *next_structural_index = idx; next_structural_index++; } -simdjson_really_inline bool is_continuation(uint8_t c) { - return (c & 0b11000000) == 0b10000000; +simdjson_inline bool is_continuation(uint8_t c) { + return (c & 0xc0) == 0x80; } -simdjson_really_inline void validate_utf8_character() { +simdjson_inline void validate_utf8_character() { // Continuation - if (simdjson_unlikely((buf[idx] & 0b01000000) == 0)) { + if (simdjson_unlikely((buf[idx] & 0x40) == 0)) { // extra continuation error = UTF8_ERROR; idx++; @@ -5156,7 +5311,7 @@ simdjson_really_inline void validate_utf8_character() { } // 2-byte - if ((buf[idx] & 0b00100000) == 0) { + if ((buf[idx] & 0x20) == 0) { // missing continuation if (simdjson_unlikely(idx+1 > len || !is_continuation(buf[idx+1]))) { if (idx+1 > len && is_streaming(partial)) { idx = len; return; } @@ -5165,13 +5320,13 @@ simdjson_really_inline void validate_utf8_character() { return; } // overlong: 1100000_ 10______ - if (buf[idx] <= 0b11000001) { error = UTF8_ERROR; } + if (buf[idx] <= 0xc1) { error = UTF8_ERROR; } idx += 2; return; } // 3-byte - if ((buf[idx] & 0b00010000) == 0) { + if ((buf[idx] & 0x10) == 0) { // missing continuation if (simdjson_unlikely(idx+2 > len || !is_continuation(buf[idx+1]) || !is_continuation(buf[idx+2]))) { if (idx+2 > len && is_streaming(partial)) { idx = len; return; } @@ -5180,9 +5335,9 @@ simdjson_really_inline void validate_utf8_character() { return; } // overlong: 11100000 100_____ ________ - if (buf[idx] == 0b11100000 && buf[idx+1] <= 0b10011111) { error = UTF8_ERROR; } + if (buf[idx] == 0xe0 && buf[idx+1] <= 0x9f) { error = UTF8_ERROR; } // surrogates: U+D800-U+DFFF 11101101 101_____ - if (buf[idx] == 0b11101101 && buf[idx+1] >= 0b10100000) { error = UTF8_ERROR; } + if (buf[idx] == 0xed && buf[idx+1] >= 0xa0) { error = UTF8_ERROR; } idx += 3; return; } @@ -5196,24 +5351,24 @@ simdjson_really_inline void validate_utf8_character() { return; } // overlong: 11110000 1000____ ________ ________ - if (buf[idx] == 0b11110000 && buf[idx+1] <= 0b10001111) { error = UTF8_ERROR; } + if (buf[idx] == 0xf0 && buf[idx+1] <= 0x8f) { error = UTF8_ERROR; } // too large: > U+10FFFF: // 11110100 (1001|101_)____ // 1111(1___|011_|0101) 10______ // also includes 5, 6, 7 and 8 byte characters: // 11111___ - if (buf[idx] == 0b11110100 && buf[idx+1] >= 0b10010000) { error = UTF8_ERROR; } - if (buf[idx] >= 0b11110101) { error = UTF8_ERROR; } + if (buf[idx] == 0xf4 && buf[idx+1] >= 0x90) { error = UTF8_ERROR; } + if (buf[idx] >= 0xf5) { error = UTF8_ERROR; } idx += 4; } // Returns true if the string is unclosed. -simdjson_really_inline bool validate_string() { +simdjson_inline bool validate_string() { idx++; // skip first quote while (idx < len && buf[idx] != '"') { if (buf[idx] == '\\') { idx += 2; - } else if (simdjson_unlikely(buf[idx] & 0b10000000)) { + } else if (simdjson_unlikely(buf[idx] & 0x80)) { validate_utf8_character(); } else { if (buf[idx] < 0x20) { error = UNESCAPED_CHARS; } @@ -5224,7 +5379,7 @@ simdjson_really_inline bool validate_string() { return false; } -simdjson_really_inline bool is_whitespace_or_operator(uint8_t c) { +simdjson_inline bool is_whitespace_or_operator(uint8_t c) { switch (c) { case '{': case '}': case '[': case ']': case ',': case ':': case ' ': case '\r': case '\n': case '\t': @@ -5237,7 +5392,7 @@ simdjson_really_inline bool is_whitespace_or_operator(uint8_t c) { // // Parse the entire input in STEP_SIZE-byte chunks. // -simdjson_really_inline error_code scan() { +simdjson_inline error_code scan() { bool unclosed_string = false; for (;idx len) { return false; } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; } + if ((data[pos + 1] & 0xc0) != 0x80) { return false; } // range check - code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); + code_point = (byte & 0x1f) << 6 | (data[pos + 1] & 0x3f); if (code_point < 0x80 || 0x7ff < code_point) { return false; } - } else if ((byte & 0b11110000) == 0b11100000) { + } else if ((byte & 0xf0) == 0xe0) { next_pos = pos + 3; if (next_pos > len) { return false; } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; } + if ((data[pos + 1] & 0xc0) != 0x80) { return false; } + if ((data[pos + 2] & 0xc0) != 0x80) { return false; } // range check - code_point = (byte & 0b00001111) << 12 | - (data[pos + 1] & 0b00111111) << 6 | - (data[pos + 2] & 0b00111111); + code_point = (byte & 0x0f) << 12 | + (data[pos + 1] & 0x3f) << 6 | + (data[pos + 2] & 0x3f); if (code_point < 0x800 || 0xffff < code_point || (0xd7ff < code_point && code_point < 0xe000)) { return false; } - } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 + } else if ((byte & 0xf8) == 0xf0) { // 0b11110000 next_pos = pos + 4; if (next_pos > len) { return false; } - if ((data[pos + 1] & 0b11000000) != 0b10000000) { return false; } - if ((data[pos + 2] & 0b11000000) != 0b10000000) { return false; } - if ((data[pos + 3] & 0b11000000) != 0b10000000) { return false; } + if ((data[pos + 1] & 0xc0) != 0x80) { return false; } + if ((data[pos + 2] & 0xc0) != 0x80) { return false; } + if ((data[pos + 3] & 0xc0) != 0x80) { return false; } // range check code_point = - (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | - (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); + (byte & 0x07) << 18 | (data[pos + 1] & 0x3f) << 12 | + (data[pos + 2] & 0x3f) << 6 | (data[pos + 3] & 0x3f); if (code_point <= 0xffff || 0x10ffff < code_point) { return false; } } else { // we may have a continuation @@ -5462,6 +5617,151 @@ simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t // // Stage 2 // +/* begin file src/generic/stage2/stringparsing.h */ +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { +namespace fallback { +namespace { +/// @private +namespace stringparsing { + +// begin copypasta +// These chars yield themselves: " \ / +// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + return false; + } + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + return false; + } + + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + return false; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_nullptrptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +} // namespace stringparsing +} // unnamed namespace +} // namespace fallback +} // namespace simdjson +/* end file src/generic/stage2/stringparsing.h */ /* begin file src/generic/stage2/tape_builder.h */ /* begin file src/generic/stage2/json_iterator.h */ /* begin file src/generic/stage2/logger.h */ @@ -5487,7 +5787,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static simdjson_really_inline char printable_char(char c) { + static simdjson_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -5496,7 +5796,7 @@ namespace logger { } // Print the header and set up log_start - static simdjson_really_inline void log_start() { + static simdjson_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -5505,7 +5805,7 @@ namespace logger { } } - simdjson_unused static simdjson_really_inline void log_string(const char *message) { + simdjson_unused static simdjson_inline void log_string(const char *message) { if (LOG_ENABLED) { printf("%s\n", message); } @@ -5513,7 +5813,7 @@ namespace logger { // Logs a single line from the stage 2 DOM parser template - static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -5591,14 +5891,14 @@ class json_iterator { * - increment_count(iter) - each time a value is found in an array or object. */ template - simdjson_warn_unused simdjson_really_inline error_code walk_document(V &visitor) noexcept; + simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -5607,7 +5907,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *peek() const noexcept; + simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -5615,56 +5915,56 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *advance() noexcept; + simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - simdjson_really_inline size_t remaining_len() const noexcept; + simdjson_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - simdjson_really_inline bool at_eof() const noexcept; + simdjson_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - simdjson_really_inline bool at_beginning() const noexcept; - simdjson_really_inline uint8_t last_structural() const noexcept; + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_value(const char *type) const noexcept; + simdjson_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_start_value(const char *type) const noexcept; + simdjson_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_end_value(const char *type) const noexcept; + simdjson_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_error(const char *error) const noexcept; + simdjson_inline void log_error(const char *error) const noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -5789,52 +6089,52 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_docum } // walk_document() -simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -simdjson_really_inline bool json_iterator::at_eof() const noexcept { +simdjson_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -simdjson_really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -5850,7 +6150,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root } } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -5882,18 +6182,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - simdjson_really_inline void append_s64(int64_t value) noexcept; + simdjson_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - simdjson_really_inline void append_u64(uint64_t value) noexcept; + simdjson_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - simdjson_really_inline void append_double(double value) noexcept; + simdjson_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -5901,24 +6201,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - simdjson_really_inline void skip() noexcept; + simdjson_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - simdjson_really_inline void skip_large_integer() noexcept; + simdjson_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - simdjson_really_inline void skip_double() noexcept; + simdjson_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -5926,50 +6226,50 @@ struct tape_writer { * all 64 bits, such as double and uint64_t. */ template - simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -simdjson_really_inline void tape_writer::append_double(double value) noexcept { +simdjson_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -simdjson_really_inline void tape_writer::skip() noexcept { +simdjson_inline void tape_writer::skip() noexcept { next_tape_loc++; } -simdjson_really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::skip_double() noexcept { +simdjson_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -5986,40 +6286,40 @@ namespace stage2 { struct tape_builder { template - simdjson_warn_unused static simdjson_really_inline error_code parse_document( + simdjson_warn_unused static simdjson_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - simdjson_warn_unused simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -6028,22 +6328,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -6051,18 +6351,18 @@ struct tape_builder { /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_really_inline tape_builder(dom::document &doc) noexcept; + simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_really_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_inline void start_container(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_document( +simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -6071,56 +6371,56 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_docum return iter.walk_document(builder); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value+1, dst); @@ -6132,16 +6432,16 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_strin return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -6163,42 +6463,42 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ return error; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -6207,24 +6507,24 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ // private: -simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -6237,13 +6537,13 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_contain return SUCCESS; } -simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -6272,6 +6572,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { + return fallback::stringparsing::parse_string(src, dst); +} + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } @@ -6336,25 +6640,25 @@ namespace { using namespace simd; struct json_character_block { - static simdjson_really_inline json_character_block classify(const simd::simd8x64& in); + static simdjson_inline json_character_block classify(const simd::simd8x64& in); // ASCII white-space ('\r','\n','\t',' ') - simdjson_really_inline uint64_t whitespace() const noexcept; + simdjson_inline uint64_t whitespace() const noexcept; // non-quote structural characters (comma, colon, braces, brackets) - simdjson_really_inline uint64_t op() const noexcept; + simdjson_inline uint64_t op() const noexcept; // neither a structural character nor a white-space, so letters, numbers and quotes - simdjson_really_inline uint64_t scalar() const noexcept; + simdjson_inline uint64_t scalar() const noexcept; uint64_t _whitespace; // ASCII white-space ('\r','\n','\t',' ') uint64_t _op; // structural characters (comma, colon, braces, brackets but not quotes) }; -simdjson_really_inline uint64_t json_character_block::whitespace() const noexcept { return _whitespace; } -simdjson_really_inline uint64_t json_character_block::op() const noexcept { return _op; } -simdjson_really_inline uint64_t json_character_block::scalar() const noexcept { return ~(op() | whitespace()); } +simdjson_inline uint64_t json_character_block::whitespace() const noexcept { return _whitespace; } +simdjson_inline uint64_t json_character_block::op() const noexcept { return _op; } +simdjson_inline uint64_t json_character_block::scalar() const noexcept { return ~(op() | whitespace()); } // This identifies structural characters (comma, colon, braces, brackets), // and ASCII white-space ('\r','\n','\t',' '). -simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { +simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. const auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); @@ -6404,21 +6708,21 @@ simdjson_really_inline json_character_block json_character_block::classify(const return { whitespace, op }; } -simdjson_really_inline bool is_ascii(const simd8x64& input) { +simdjson_inline bool is_ascii(const simd8x64& input) { return input.reduce_or().is_ascii(); } -simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 +simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 +simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } @@ -6435,7 +6739,7 @@ namespace utf8_validation { using namespace simd; - simdjson_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -6525,7 +6829,7 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - simdjson_really_inline simd8 check_multibyte_lengths(const simd8 input, + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -6538,7 +6842,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - simdjson_really_inline simd8 is_incomplete(const simd8 input) { + simdjson_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ #if SIMDJSON_IMPLEMENTATION_ICELAKE @@ -6550,14 +6854,14 @@ using namespace simd; 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #else static const uint8_t max_array[32] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #endif const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); @@ -6575,7 +6879,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - simdjson_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -6586,13 +6890,21 @@ using namespace simd; // The only problem that can happen at EOF is that a multibyte character is too short // or a byte value too large in the last bytes: check_special_cases only checks for bytes // too large in the first of two bytes. - simdjson_really_inline void check_eof() { + simdjson_inline void check_eof() { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. this->error |= this->prev_incomplete; } - simdjson_really_inline void check_next_input(const simd8x64& input) { +#ifndef SIMDJSON_IF_CONSTEXPR +#if SIMDJSON_CPLUSPLUS17 +#define SIMDJSON_IF_CONSTEXPR if constexpr +#else +#define SIMDJSON_IF_CONSTEXPR if +#endif +#endif + + simdjson_inline void check_next_input(const simd8x64& input) { if(simdjson_likely(is_ascii(input))) { this->error |= this->prev_incomplete; } else { @@ -6601,12 +6913,12 @@ using namespace simd; ||(simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support one, two or four chunks per 64-byte block."); - if(simd8x64::NUM_CHUNKS == 1) { + SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - } if(simd8x64::NUM_CHUNKS == 2) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if(simd8x64::NUM_CHUNKS == 4) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -6617,7 +6929,7 @@ using namespace simd; } } // do not forget to call check_eof! - simdjson_really_inline error_code errors() { + simdjson_inline error_code errors() { return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; } @@ -6647,10 +6959,10 @@ namespace { template struct buf_block_reader { public: - simdjson_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_really_inline size_t block_index(); - simdjson_really_inline bool has_full_block() const; - simdjson_really_inline const uint8_t *full_block() const; + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; /** * Get the last block, padded with spaces. * @@ -6660,8 +6972,8 @@ struct buf_block_reader { * * @return the number of effective characters in the last block. */ - simdjson_really_inline size_t get_remainder(uint8_t *dst) const; - simdjson_really_inline void advance(); + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); private: const uint8_t *buf; const size_t len; @@ -6700,23 +7012,23 @@ simdjson_unused static char * format_mask(uint64_t mask) { } template -simdjson_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} template -simdjson_really_inline size_t buf_block_reader::block_index() { return idx; } +simdjson_inline size_t buf_block_reader::block_index() { return idx; } template -simdjson_really_inline bool buf_block_reader::has_full_block() const { +simdjson_inline bool buf_block_reader::has_full_block() const { return idx < lenminusstep; } template -simdjson_really_inline const uint8_t *buf_block_reader::full_block() const { +simdjson_inline const uint8_t *buf_block_reader::full_block() const { return &buf[idx]; } template -simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. std::memcpy(dst, buf + idx, len - idx); @@ -6724,7 +7036,7 @@ simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t } template -simdjson_really_inline void buf_block_reader::advance() { +simdjson_inline void buf_block_reader::advance() { idx += STEP_SIZE; } @@ -6740,27 +7052,27 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : + simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } + simdjson_inline uint64_t escaped() const { return _escaped; } // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_really_inline uint64_t escape() const { return _backslash & ~_escaped; } + simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } + simdjson_inline uint64_t quote() const { return _quote; } // Start quotes of strings - simdjson_really_inline uint64_t string_start() const { return _quote & _in_string; } + simdjson_inline uint64_t string_start() const { return _quote & _in_string; } // End quotes of strings - simdjson_really_inline uint64_t string_end() const { return _quote & ~_in_string; } + simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } // backslash characters uint64_t _backslash; @@ -6775,14 +7087,14 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); + simdjson_inline json_string_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Intended to be defined by the implementation - simdjson_really_inline uint64_t find_escaped(uint64_t escape); - simdjson_really_inline uint64_t find_escaped_branchless(uint64_t escape); + simdjson_inline uint64_t find_escaped(uint64_t escape); + simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -6817,7 +7129,7 @@ class json_string_scanner { // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -6842,7 +7154,7 @@ simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uin // // Backslash sequences outside of quotes will be detected in stage 2. // -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -6875,7 +7187,7 @@ simdjson_really_inline json_string_block json_string_scanner::next(const simd::s ); } -simdjson_really_inline error_code json_string_scanner::finish() { +simdjson_inline error_code json_string_scanner::finish() { if (prev_in_string) { return UNCLOSED_STRING; } @@ -6913,25 +7225,25 @@ namespace stage1 { struct json_block { public: // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_really_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} /** * The start of structurals. * In simdjson prior to v0.3, these were called the pseudo-structural characters. **/ - simdjson_really_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } /** All JSON whitespace (i.e. not in a string) */ - simdjson_really_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } // Helpers /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } // string and escape characters json_string_block _string; @@ -6946,12 +7258,12 @@ struct json_block { * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". * They may reside inside a string. **/ - simdjson_really_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } /** * The start of non-operator runs, like 123, true and "abc". * It main reside inside a string. **/ - simdjson_really_inline uint64_t potential_scalar_start() const noexcept { + simdjson_inline uint64_t potential_scalar_start() const noexcept { // The term "scalar" refers to anything except structural characters and white space // (so letters, numbers, quotes). // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space @@ -6962,7 +7274,7 @@ struct json_block { * Whether the given character is immediately after a non-operator like 123, true. * The characters following a quote are not included. */ - simdjson_really_inline uint64_t follows_potential_scalar() const noexcept { + simdjson_inline uint64_t follows_potential_scalar() const noexcept { // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a // white space. @@ -6986,10 +7298,10 @@ struct json_block { */ class json_scanner { public: - json_scanner() {} - simdjson_really_inline json_block next(const simd::simd8x64& in); + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Whether the last character of the previous iteration is part of a scalar token @@ -7006,13 +7318,13 @@ class json_scanner { // // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); // -simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { +simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { const uint64_t result = match << 1 | overflow; overflow = match >> 63; return result; } -simdjson_really_inline json_block json_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); // identifies the white-space and the structural characters json_character_block characters = json_character_block::classify(in); @@ -7037,7 +7349,7 @@ simdjson_really_inline json_block json_scanner::next(const simd::simd8x64 - simdjson_really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); + simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -simdjson_really_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { +simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { uint64_t mask = block.whitespace(); dst += in.compress(mask, dst); } -simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { +simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { error_code error = scanner.finish(); if (error) { dst_len = 0; return error; } dst_len = dst - dst_start; @@ -7087,7 +7399,7 @@ simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size } template<> -simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block_buf); simd::simd8x64 in_2(block_buf+64); json_block block_1 = scanner.next(in_1); @@ -7098,7 +7410,7 @@ simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, b } template<> -simdjson_really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block_buf); json_block block_1 = scanner.next(in_1); this->next(block_buf, block_1); @@ -7171,7 +7483,7 @@ namespace { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -7246,7 +7558,7 @@ class bit_indexer { public: uint32_t *tail; - simdjson_really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at @@ -7257,9 +7569,9 @@ class bit_indexer { // If the kernel sets SIMDJSON_CUSTOM_BIT_INDEXER, then it will provide its own // version of the code. #ifdef SIMDJSON_CUSTOM_BIT_INDEXER - simdjson_really_inline void write(uint32_t idx, uint64_t bits); + simdjson_inline void write(uint32_t idx, uint64_t bits); #else - simdjson_really_inline void write(uint32_t idx, uint64_t bits) { + simdjson_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -7368,11 +7680,11 @@ class json_structural_indexer { static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; private: - simdjson_really_inline json_structural_indexer(uint32_t *structural_indexes); + simdjson_inline json_structural_indexer(uint32_t *structural_indexes); template - simdjson_really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); + simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); json_scanner scanner{}; utf8_checker checker{}; @@ -7381,26 +7693,26 @@ class json_structural_indexer { uint64_t unescaped_chars_error = 0; }; -simdjson_really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} +simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} // Skip the last character if it is partial -simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { +simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { if (simdjson_unlikely(len < 3)) { switch (len) { case 2: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 2 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left return len; case 1: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left return len; case 0: return len; } } - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left return len; } @@ -7449,7 +7761,7 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_pa } template<> -simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block); simd::simd8x64 in_2(block+64); json_block block_1 = scanner.next(in_1); @@ -7460,14 +7772,14 @@ simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *bl } template<> -simdjson_really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block); json_block block_1 = scanner.next(in_1); this->next(in_1, block_1, reader.block_index()); reader.advance(); } -simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { +simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -7475,7 +7787,7 @@ simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64 backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + return false; + } + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + return false; + } + + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + return false; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_nullptrptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +} // namespace stringparsing +} // unnamed namespace +} // namespace icelake +} // namespace simdjson +/* end file src/generic/stage2/stringparsing.h */ /* begin file src/generic/stage2/tape_builder.h */ /* begin file src/generic/stage2/json_iterator.h */ /* begin file src/generic/stage2/logger.h */ @@ -7693,7 +8150,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static simdjson_really_inline char printable_char(char c) { + static simdjson_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -7702,7 +8159,7 @@ namespace logger { } // Print the header and set up log_start - static simdjson_really_inline void log_start() { + static simdjson_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -7711,7 +8168,7 @@ namespace logger { } } - simdjson_unused static simdjson_really_inline void log_string(const char *message) { + simdjson_unused static simdjson_inline void log_string(const char *message) { if (LOG_ENABLED) { printf("%s\n", message); } @@ -7719,7 +8176,7 @@ namespace logger { // Logs a single line from the stage 2 DOM parser template - static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -7797,14 +8254,14 @@ class json_iterator { * - increment_count(iter) - each time a value is found in an array or object. */ template - simdjson_warn_unused simdjson_really_inline error_code walk_document(V &visitor) noexcept; + simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -7813,7 +8270,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *peek() const noexcept; + simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -7821,56 +8278,56 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *advance() noexcept; + simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - simdjson_really_inline size_t remaining_len() const noexcept; + simdjson_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - simdjson_really_inline bool at_eof() const noexcept; + simdjson_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - simdjson_really_inline bool at_beginning() const noexcept; - simdjson_really_inline uint8_t last_structural() const noexcept; + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_value(const char *type) const noexcept; + simdjson_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_start_value(const char *type) const noexcept; + simdjson_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_end_value(const char *type) const noexcept; + simdjson_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_error(const char *error) const noexcept; + simdjson_inline void log_error(const char *error) const noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -7995,52 +8452,52 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_docum } // walk_document() -simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -simdjson_really_inline bool json_iterator::at_eof() const noexcept { +simdjson_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -simdjson_really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -8056,7 +8513,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root } } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -8088,18 +8545,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - simdjson_really_inline void append_s64(int64_t value) noexcept; + simdjson_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - simdjson_really_inline void append_u64(uint64_t value) noexcept; + simdjson_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - simdjson_really_inline void append_double(double value) noexcept; + simdjson_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -8107,24 +8564,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - simdjson_really_inline void skip() noexcept; + simdjson_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - simdjson_really_inline void skip_large_integer() noexcept; + simdjson_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - simdjson_really_inline void skip_double() noexcept; + simdjson_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -8132,50 +8589,50 @@ struct tape_writer { * all 64 bits, such as double and uint64_t. */ template - simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -simdjson_really_inline void tape_writer::append_double(double value) noexcept { +simdjson_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -simdjson_really_inline void tape_writer::skip() noexcept { +simdjson_inline void tape_writer::skip() noexcept { next_tape_loc++; } -simdjson_really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::skip_double() noexcept { +simdjson_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -8192,40 +8649,40 @@ namespace stage2 { struct tape_builder { template - simdjson_warn_unused static simdjson_really_inline error_code parse_document( + simdjson_warn_unused static simdjson_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - simdjson_warn_unused simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -8234,22 +8691,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -8257,18 +8714,18 @@ struct tape_builder { /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_really_inline tape_builder(dom::document &doc) noexcept; + simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_really_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_inline void start_container(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_document( +simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -8277,56 +8734,56 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_docum return iter.walk_document(builder); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value+1, dst); @@ -8338,16 +8795,16 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_strin return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -8369,42 +8826,42 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ return error; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -8413,24 +8870,24 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ // private: -simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -8443,13 +8900,13 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_contain return SUCCESS; } -simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -8475,7 +8932,7 @@ namespace icelake { namespace { namespace stage1 { -simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } return find_escaped_branchless(backslash); } @@ -8505,6 +8962,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { + return icelake::stringparsing::parse_string(src, dst); +} + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } @@ -8570,25 +9031,25 @@ namespace { using namespace simd; struct json_character_block { - static simdjson_really_inline json_character_block classify(const simd::simd8x64& in); + static simdjson_inline json_character_block classify(const simd::simd8x64& in); // ASCII white-space ('\r','\n','\t',' ') - simdjson_really_inline uint64_t whitespace() const noexcept; + simdjson_inline uint64_t whitespace() const noexcept; // non-quote structural characters (comma, colon, braces, brackets) - simdjson_really_inline uint64_t op() const noexcept; + simdjson_inline uint64_t op() const noexcept; // neither a structural character nor a white-space, so letters, numbers and quotes - simdjson_really_inline uint64_t scalar() const noexcept; + simdjson_inline uint64_t scalar() const noexcept; uint64_t _whitespace; // ASCII white-space ('\r','\n','\t',' ') uint64_t _op; // structural characters (comma, colon, braces, brackets but not quotes) }; -simdjson_really_inline uint64_t json_character_block::whitespace() const noexcept { return _whitespace; } -simdjson_really_inline uint64_t json_character_block::op() const noexcept { return _op; } -simdjson_really_inline uint64_t json_character_block::scalar() const noexcept { return ~(op() | whitespace()); } +simdjson_inline uint64_t json_character_block::whitespace() const noexcept { return _whitespace; } +simdjson_inline uint64_t json_character_block::op() const noexcept { return _op; } +simdjson_inline uint64_t json_character_block::scalar() const noexcept { return ~(op() | whitespace()); } // This identifies structural characters (comma, colon, braces, brackets), // and ASCII white-space ('\r','\n','\t',' '). -simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { +simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. const auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); @@ -8641,21 +9102,21 @@ simdjson_really_inline json_character_block json_character_block::classify(const return { whitespace, op }; } -simdjson_really_inline bool is_ascii(const simd8x64& input) { +simdjson_inline bool is_ascii(const simd8x64& input) { return input.reduce_or().is_ascii(); } -simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 +simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 +simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } @@ -8672,7 +9133,7 @@ namespace utf8_validation { using namespace simd; - simdjson_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -8762,7 +9223,7 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - simdjson_really_inline simd8 check_multibyte_lengths(const simd8 input, + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -8775,7 +9236,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - simdjson_really_inline simd8 is_incomplete(const simd8 input) { + simdjson_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ #if SIMDJSON_IMPLEMENTATION_ICELAKE @@ -8787,14 +9248,14 @@ using namespace simd; 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #else static const uint8_t max_array[32] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #endif const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); @@ -8812,7 +9273,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - simdjson_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -8823,13 +9284,21 @@ using namespace simd; // The only problem that can happen at EOF is that a multibyte character is too short // or a byte value too large in the last bytes: check_special_cases only checks for bytes // too large in the first of two bytes. - simdjson_really_inline void check_eof() { + simdjson_inline void check_eof() { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. this->error |= this->prev_incomplete; } - simdjson_really_inline void check_next_input(const simd8x64& input) { +#ifndef SIMDJSON_IF_CONSTEXPR +#if SIMDJSON_CPLUSPLUS17 +#define SIMDJSON_IF_CONSTEXPR if constexpr +#else +#define SIMDJSON_IF_CONSTEXPR if +#endif +#endif + + simdjson_inline void check_next_input(const simd8x64& input) { if(simdjson_likely(is_ascii(input))) { this->error |= this->prev_incomplete; } else { @@ -8838,12 +9307,12 @@ using namespace simd; ||(simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support one, two or four chunks per 64-byte block."); - if(simd8x64::NUM_CHUNKS == 1) { + SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - } if(simd8x64::NUM_CHUNKS == 2) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if(simd8x64::NUM_CHUNKS == 4) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -8854,7 +9323,7 @@ using namespace simd; } } // do not forget to call check_eof! - simdjson_really_inline error_code errors() { + simdjson_inline error_code errors() { return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; } @@ -8882,10 +9351,10 @@ namespace { template struct buf_block_reader { public: - simdjson_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_really_inline size_t block_index(); - simdjson_really_inline bool has_full_block() const; - simdjson_really_inline const uint8_t *full_block() const; + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; /** * Get the last block, padded with spaces. * @@ -8895,8 +9364,8 @@ struct buf_block_reader { * * @return the number of effective characters in the last block. */ - simdjson_really_inline size_t get_remainder(uint8_t *dst) const; - simdjson_really_inline void advance(); + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); private: const uint8_t *buf; const size_t len; @@ -8935,23 +9404,23 @@ simdjson_unused static char * format_mask(uint64_t mask) { } template -simdjson_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} template -simdjson_really_inline size_t buf_block_reader::block_index() { return idx; } +simdjson_inline size_t buf_block_reader::block_index() { return idx; } template -simdjson_really_inline bool buf_block_reader::has_full_block() const { +simdjson_inline bool buf_block_reader::has_full_block() const { return idx < lenminusstep; } template -simdjson_really_inline const uint8_t *buf_block_reader::full_block() const { +simdjson_inline const uint8_t *buf_block_reader::full_block() const { return &buf[idx]; } template -simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. std::memcpy(dst, buf + idx, len - idx); @@ -8959,7 +9428,7 @@ simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t } template -simdjson_really_inline void buf_block_reader::advance() { +simdjson_inline void buf_block_reader::advance() { idx += STEP_SIZE; } @@ -8975,27 +9444,27 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : + simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } + simdjson_inline uint64_t escaped() const { return _escaped; } // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_really_inline uint64_t escape() const { return _backslash & ~_escaped; } + simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } + simdjson_inline uint64_t quote() const { return _quote; } // Start quotes of strings - simdjson_really_inline uint64_t string_start() const { return _quote & _in_string; } + simdjson_inline uint64_t string_start() const { return _quote & _in_string; } // End quotes of strings - simdjson_really_inline uint64_t string_end() const { return _quote & ~_in_string; } + simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } // backslash characters uint64_t _backslash; @@ -9010,14 +9479,14 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); + simdjson_inline json_string_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Intended to be defined by the implementation - simdjson_really_inline uint64_t find_escaped(uint64_t escape); - simdjson_really_inline uint64_t find_escaped_branchless(uint64_t escape); + simdjson_inline uint64_t find_escaped(uint64_t escape); + simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -9052,7 +9521,7 @@ class json_string_scanner { // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -9077,7 +9546,7 @@ simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uin // // Backslash sequences outside of quotes will be detected in stage 2. // -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -9110,7 +9579,7 @@ simdjson_really_inline json_string_block json_string_scanner::next(const simd::s ); } -simdjson_really_inline error_code json_string_scanner::finish() { +simdjson_inline error_code json_string_scanner::finish() { if (prev_in_string) { return UNCLOSED_STRING; } @@ -9148,25 +9617,25 @@ namespace stage1 { struct json_block { public: // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_really_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} /** * The start of structurals. * In simdjson prior to v0.3, these were called the pseudo-structural characters. **/ - simdjson_really_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } /** All JSON whitespace (i.e. not in a string) */ - simdjson_really_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } // Helpers /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } // string and escape characters json_string_block _string; @@ -9181,12 +9650,12 @@ struct json_block { * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". * They may reside inside a string. **/ - simdjson_really_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } /** * The start of non-operator runs, like 123, true and "abc". * It main reside inside a string. **/ - simdjson_really_inline uint64_t potential_scalar_start() const noexcept { + simdjson_inline uint64_t potential_scalar_start() const noexcept { // The term "scalar" refers to anything except structural characters and white space // (so letters, numbers, quotes). // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space @@ -9197,7 +9666,7 @@ struct json_block { * Whether the given character is immediately after a non-operator like 123, true. * The characters following a quote are not included. */ - simdjson_really_inline uint64_t follows_potential_scalar() const noexcept { + simdjson_inline uint64_t follows_potential_scalar() const noexcept { // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a // white space. @@ -9221,10 +9690,10 @@ struct json_block { */ class json_scanner { public: - json_scanner() {} - simdjson_really_inline json_block next(const simd::simd8x64& in); + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Whether the last character of the previous iteration is part of a scalar token @@ -9241,13 +9710,13 @@ class json_scanner { // // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); // -simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { +simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { const uint64_t result = match << 1 | overflow; overflow = match >> 63; return result; } -simdjson_really_inline json_block json_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); // identifies the white-space and the structural characters json_character_block characters = json_character_block::classify(in); @@ -9272,7 +9741,7 @@ simdjson_really_inline json_block json_scanner::next(const simd::simd8x64 - simdjson_really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); + simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -simdjson_really_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { +simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { uint64_t mask = block.whitespace(); dst += in.compress(mask, dst); } -simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { +simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { error_code error = scanner.finish(); if (error) { dst_len = 0; return error; } dst_len = dst - dst_start; @@ -9322,7 +9791,7 @@ simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size } template<> -simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block_buf); simd::simd8x64 in_2(block_buf+64); json_block block_1 = scanner.next(in_1); @@ -9333,7 +9802,7 @@ simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, b } template<> -simdjson_really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block_buf); json_block block_1 = scanner.next(in_1); this->next(block_buf, block_1); @@ -9406,7 +9875,7 @@ namespace { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -9481,7 +9950,7 @@ class bit_indexer { public: uint32_t *tail; - simdjson_really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at @@ -9492,9 +9961,9 @@ class bit_indexer { // If the kernel sets SIMDJSON_CUSTOM_BIT_INDEXER, then it will provide its own // version of the code. #ifdef SIMDJSON_CUSTOM_BIT_INDEXER - simdjson_really_inline void write(uint32_t idx, uint64_t bits); + simdjson_inline void write(uint32_t idx, uint64_t bits); #else - simdjson_really_inline void write(uint32_t idx, uint64_t bits) { + simdjson_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -9603,11 +10072,11 @@ class json_structural_indexer { static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; private: - simdjson_really_inline json_structural_indexer(uint32_t *structural_indexes); + simdjson_inline json_structural_indexer(uint32_t *structural_indexes); template - simdjson_really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); + simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); json_scanner scanner{}; utf8_checker checker{}; @@ -9616,26 +10085,26 @@ class json_structural_indexer { uint64_t unescaped_chars_error = 0; }; -simdjson_really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} +simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} // Skip the last character if it is partial -simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { +simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { if (simdjson_unlikely(len < 3)) { switch (len) { case 2: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 2 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left return len; case 1: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left return len; case 0: return len; } } - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left return len; } @@ -9684,7 +10153,7 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_pa } template<> -simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block); simd::simd8x64 in_2(block+64); json_block block_1 = scanner.next(in_1); @@ -9695,14 +10164,14 @@ simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *bl } template<> -simdjson_really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block); json_block block_1 = scanner.next(in_1); this->next(in_1, block_1, reader.block_index()); reader.advance(); } -simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { +simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -9710,7 +10179,7 @@ simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64 backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + return false; + } + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + return false; + } + + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + return false; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_nullptrptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +} // namespace stringparsing +} // unnamed namespace +} // namespace haswell +} // namespace simdjson +/* end file src/generic/stage2/stringparsing.h */ /* begin file src/generic/stage2/tape_builder.h */ /* begin file src/generic/stage2/json_iterator.h */ /* begin file src/generic/stage2/logger.h */ @@ -9881,7 +10495,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static simdjson_really_inline char printable_char(char c) { + static simdjson_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -9890,7 +10504,7 @@ namespace logger { } // Print the header and set up log_start - static simdjson_really_inline void log_start() { + static simdjson_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -9899,7 +10513,7 @@ namespace logger { } } - simdjson_unused static simdjson_really_inline void log_string(const char *message) { + simdjson_unused static simdjson_inline void log_string(const char *message) { if (LOG_ENABLED) { printf("%s\n", message); } @@ -9907,7 +10521,7 @@ namespace logger { // Logs a single line from the stage 2 DOM parser template - static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -9985,14 +10599,14 @@ class json_iterator { * - increment_count(iter) - each time a value is found in an array or object. */ template - simdjson_warn_unused simdjson_really_inline error_code walk_document(V &visitor) noexcept; + simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -10001,7 +10615,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *peek() const noexcept; + simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -10009,56 +10623,56 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *advance() noexcept; + simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - simdjson_really_inline size_t remaining_len() const noexcept; + simdjson_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - simdjson_really_inline bool at_eof() const noexcept; + simdjson_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - simdjson_really_inline bool at_beginning() const noexcept; - simdjson_really_inline uint8_t last_structural() const noexcept; + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_value(const char *type) const noexcept; + simdjson_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_start_value(const char *type) const noexcept; + simdjson_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_end_value(const char *type) const noexcept; + simdjson_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_error(const char *error) const noexcept; + simdjson_inline void log_error(const char *error) const noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -10183,52 +10797,52 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_docum } // walk_document() -simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -simdjson_really_inline bool json_iterator::at_eof() const noexcept { +simdjson_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -simdjson_really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -10244,7 +10858,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root } } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -10276,18 +10890,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - simdjson_really_inline void append_s64(int64_t value) noexcept; + simdjson_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - simdjson_really_inline void append_u64(uint64_t value) noexcept; + simdjson_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - simdjson_really_inline void append_double(double value) noexcept; + simdjson_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -10295,24 +10909,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - simdjson_really_inline void skip() noexcept; + simdjson_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - simdjson_really_inline void skip_large_integer() noexcept; + simdjson_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - simdjson_really_inline void skip_double() noexcept; + simdjson_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -10320,50 +10934,50 @@ struct tape_writer { * all 64 bits, such as double and uint64_t. */ template - simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -simdjson_really_inline void tape_writer::append_double(double value) noexcept { +simdjson_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -simdjson_really_inline void tape_writer::skip() noexcept { +simdjson_inline void tape_writer::skip() noexcept { next_tape_loc++; } -simdjson_really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::skip_double() noexcept { +simdjson_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -10380,40 +10994,40 @@ namespace stage2 { struct tape_builder { template - simdjson_warn_unused static simdjson_really_inline error_code parse_document( + simdjson_warn_unused static simdjson_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - simdjson_warn_unused simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -10422,22 +11036,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -10445,18 +11059,18 @@ struct tape_builder { /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_really_inline tape_builder(dom::document &doc) noexcept; + simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_really_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_inline void start_container(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_document( +simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -10465,56 +11079,56 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_docum return iter.walk_document(builder); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value+1, dst); @@ -10526,16 +11140,16 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_strin return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -10557,42 +11171,42 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ return error; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -10601,24 +11215,24 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ // private: -simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -10631,13 +11245,13 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_contain return SUCCESS; } -simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -10663,7 +11277,7 @@ namespace haswell { namespace { namespace stage1 { -simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } return find_escaped_branchless(backslash); } @@ -10693,6 +11307,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { + return haswell::stringparsing::parse_string(src, dst); +} + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } @@ -10753,17 +11371,17 @@ namespace { using namespace simd; struct json_character_block { - static simdjson_really_inline json_character_block classify(const simd::simd8x64& in); + static simdjson_inline json_character_block classify(const simd::simd8x64& in); - simdjson_really_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_really_inline uint64_t op() const noexcept { return _op; } - simdjson_really_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } + simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } + simdjson_inline uint64_t op() const noexcept { return _op; } + simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } uint64_t _whitespace; uint64_t _op; }; -simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { +simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); @@ -10791,22 +11409,22 @@ simdjson_really_inline json_character_block json_character_block::classify(const return { whitespace, op }; } -simdjson_really_inline bool is_ascii(const simd8x64& input) { +simdjson_inline bool is_ascii(const simd8x64& input) { // careful: 0x80 is not ascii. - return input.reduce_or().saturating_sub(0b01111111u).bits_not_set_anywhere(); + return input.reduce_or().saturating_sub(0x7fu).bits_not_set_anywhere(); } -simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 +simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 +simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } @@ -10823,7 +11441,7 @@ namespace utf8_validation { using namespace simd; - simdjson_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -10913,7 +11531,7 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - simdjson_really_inline simd8 check_multibyte_lengths(const simd8 input, + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -10926,7 +11544,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - simdjson_really_inline simd8 is_incomplete(const simd8 input) { + simdjson_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ #if SIMDJSON_IMPLEMENTATION_ICELAKE @@ -10938,14 +11556,14 @@ using namespace simd; 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #else static const uint8_t max_array[32] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #endif const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); @@ -10963,7 +11581,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - simdjson_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -10974,13 +11592,21 @@ using namespace simd; // The only problem that can happen at EOF is that a multibyte character is too short // or a byte value too large in the last bytes: check_special_cases only checks for bytes // too large in the first of two bytes. - simdjson_really_inline void check_eof() { + simdjson_inline void check_eof() { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. this->error |= this->prev_incomplete; } - simdjson_really_inline void check_next_input(const simd8x64& input) { +#ifndef SIMDJSON_IF_CONSTEXPR +#if SIMDJSON_CPLUSPLUS17 +#define SIMDJSON_IF_CONSTEXPR if constexpr +#else +#define SIMDJSON_IF_CONSTEXPR if +#endif +#endif + + simdjson_inline void check_next_input(const simd8x64& input) { if(simdjson_likely(is_ascii(input))) { this->error |= this->prev_incomplete; } else { @@ -10989,12 +11615,12 @@ using namespace simd; ||(simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support one, two or four chunks per 64-byte block."); - if(simd8x64::NUM_CHUNKS == 1) { + SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - } if(simd8x64::NUM_CHUNKS == 2) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if(simd8x64::NUM_CHUNKS == 4) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -11005,7 +11631,7 @@ using namespace simd; } } // do not forget to call check_eof! - simdjson_really_inline error_code errors() { + simdjson_inline error_code errors() { return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; } @@ -11033,10 +11659,10 @@ namespace { template struct buf_block_reader { public: - simdjson_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_really_inline size_t block_index(); - simdjson_really_inline bool has_full_block() const; - simdjson_really_inline const uint8_t *full_block() const; + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; /** * Get the last block, padded with spaces. * @@ -11046,8 +11672,8 @@ struct buf_block_reader { * * @return the number of effective characters in the last block. */ - simdjson_really_inline size_t get_remainder(uint8_t *dst) const; - simdjson_really_inline void advance(); + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); private: const uint8_t *buf; const size_t len; @@ -11086,23 +11712,23 @@ simdjson_unused static char * format_mask(uint64_t mask) { } template -simdjson_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} template -simdjson_really_inline size_t buf_block_reader::block_index() { return idx; } +simdjson_inline size_t buf_block_reader::block_index() { return idx; } template -simdjson_really_inline bool buf_block_reader::has_full_block() const { +simdjson_inline bool buf_block_reader::has_full_block() const { return idx < lenminusstep; } template -simdjson_really_inline const uint8_t *buf_block_reader::full_block() const { +simdjson_inline const uint8_t *buf_block_reader::full_block() const { return &buf[idx]; } template -simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. std::memcpy(dst, buf + idx, len - idx); @@ -11110,7 +11736,7 @@ simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t } template -simdjson_really_inline void buf_block_reader::advance() { +simdjson_inline void buf_block_reader::advance() { idx += STEP_SIZE; } @@ -11126,27 +11752,27 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : + simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } + simdjson_inline uint64_t escaped() const { return _escaped; } // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_really_inline uint64_t escape() const { return _backslash & ~_escaped; } + simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } + simdjson_inline uint64_t quote() const { return _quote; } // Start quotes of strings - simdjson_really_inline uint64_t string_start() const { return _quote & _in_string; } + simdjson_inline uint64_t string_start() const { return _quote & _in_string; } // End quotes of strings - simdjson_really_inline uint64_t string_end() const { return _quote & ~_in_string; } + simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } // backslash characters uint64_t _backslash; @@ -11161,14 +11787,14 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); + simdjson_inline json_string_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Intended to be defined by the implementation - simdjson_really_inline uint64_t find_escaped(uint64_t escape); - simdjson_really_inline uint64_t find_escaped_branchless(uint64_t escape); + simdjson_inline uint64_t find_escaped(uint64_t escape); + simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -11203,7 +11829,7 @@ class json_string_scanner { // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -11228,7 +11854,7 @@ simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uin // // Backslash sequences outside of quotes will be detected in stage 2. // -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -11261,7 +11887,7 @@ simdjson_really_inline json_string_block json_string_scanner::next(const simd::s ); } -simdjson_really_inline error_code json_string_scanner::finish() { +simdjson_inline error_code json_string_scanner::finish() { if (prev_in_string) { return UNCLOSED_STRING; } @@ -11299,25 +11925,25 @@ namespace stage1 { struct json_block { public: // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_really_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} /** * The start of structurals. * In simdjson prior to v0.3, these were called the pseudo-structural characters. **/ - simdjson_really_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } /** All JSON whitespace (i.e. not in a string) */ - simdjson_really_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } // Helpers /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } // string and escape characters json_string_block _string; @@ -11332,12 +11958,12 @@ struct json_block { * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". * They may reside inside a string. **/ - simdjson_really_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } /** * The start of non-operator runs, like 123, true and "abc". * It main reside inside a string. **/ - simdjson_really_inline uint64_t potential_scalar_start() const noexcept { + simdjson_inline uint64_t potential_scalar_start() const noexcept { // The term "scalar" refers to anything except structural characters and white space // (so letters, numbers, quotes). // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space @@ -11348,7 +11974,7 @@ struct json_block { * Whether the given character is immediately after a non-operator like 123, true. * The characters following a quote are not included. */ - simdjson_really_inline uint64_t follows_potential_scalar() const noexcept { + simdjson_inline uint64_t follows_potential_scalar() const noexcept { // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a // white space. @@ -11372,10 +11998,10 @@ struct json_block { */ class json_scanner { public: - json_scanner() {} - simdjson_really_inline json_block next(const simd::simd8x64& in); + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Whether the last character of the previous iteration is part of a scalar token @@ -11392,13 +12018,13 @@ class json_scanner { // // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); // -simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { +simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { const uint64_t result = match << 1 | overflow; overflow = match >> 63; return result; } -simdjson_really_inline json_block json_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); // identifies the white-space and the structural characters json_character_block characters = json_character_block::classify(in); @@ -11423,7 +12049,7 @@ simdjson_really_inline json_block json_scanner::next(const simd::simd8x64 - simdjson_really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); + simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -simdjson_really_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { +simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { uint64_t mask = block.whitespace(); dst += in.compress(mask, dst); } -simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { +simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { error_code error = scanner.finish(); if (error) { dst_len = 0; return error; } dst_len = dst - dst_start; @@ -11473,7 +12099,7 @@ simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size } template<> -simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block_buf); simd::simd8x64 in_2(block_buf+64); json_block block_1 = scanner.next(in_1); @@ -11484,7 +12110,7 @@ simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, b } template<> -simdjson_really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block_buf); json_block block_1 = scanner.next(in_1); this->next(block_buf, block_1); @@ -11557,7 +12183,7 @@ namespace { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -11632,7 +12258,7 @@ class bit_indexer { public: uint32_t *tail; - simdjson_really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at @@ -11643,9 +12269,9 @@ class bit_indexer { // If the kernel sets SIMDJSON_CUSTOM_BIT_INDEXER, then it will provide its own // version of the code. #ifdef SIMDJSON_CUSTOM_BIT_INDEXER - simdjson_really_inline void write(uint32_t idx, uint64_t bits); + simdjson_inline void write(uint32_t idx, uint64_t bits); #else - simdjson_really_inline void write(uint32_t idx, uint64_t bits) { + simdjson_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -11754,11 +12380,11 @@ class json_structural_indexer { static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; private: - simdjson_really_inline json_structural_indexer(uint32_t *structural_indexes); + simdjson_inline json_structural_indexer(uint32_t *structural_indexes); template - simdjson_really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); + simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); json_scanner scanner{}; utf8_checker checker{}; @@ -11767,26 +12393,26 @@ class json_structural_indexer { uint64_t unescaped_chars_error = 0; }; -simdjson_really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} +simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} // Skip the last character if it is partial -simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { +simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { if (simdjson_unlikely(len < 3)) { switch (len) { case 2: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 2 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left return len; case 1: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left return len; case 0: return len; } } - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left return len; } @@ -11835,7 +12461,7 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_pa } template<> -simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block); simd::simd8x64 in_2(block+64); json_block block_1 = scanner.next(in_1); @@ -11846,14 +12472,14 @@ simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *bl } template<> -simdjson_really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block); json_block block_1 = scanner.next(in_1); this->next(in_1, block_1, reader.block_index()); reader.advance(); } -simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { +simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -11861,7 +12487,7 @@ simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64 backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + return false; + } + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + return false; + } + + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + return false; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_nullptrptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +} // namespace stringparsing +} // unnamed namespace +} // namespace ppc64 +} // namespace simdjson +/* end file src/generic/stage2/stringparsing.h */ /* begin file src/generic/stage2/tape_builder.h */ /* begin file src/generic/stage2/json_iterator.h */ /* begin file src/generic/stage2/logger.h */ @@ -12033,7 +12803,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static simdjson_really_inline char printable_char(char c) { + static simdjson_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -12042,7 +12812,7 @@ namespace logger { } // Print the header and set up log_start - static simdjson_really_inline void log_start() { + static simdjson_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -12051,7 +12821,7 @@ namespace logger { } } - simdjson_unused static simdjson_really_inline void log_string(const char *message) { + simdjson_unused static simdjson_inline void log_string(const char *message) { if (LOG_ENABLED) { printf("%s\n", message); } @@ -12059,7 +12829,7 @@ namespace logger { // Logs a single line from the stage 2 DOM parser template - static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -12137,14 +12907,14 @@ class json_iterator { * - increment_count(iter) - each time a value is found in an array or object. */ template - simdjson_warn_unused simdjson_really_inline error_code walk_document(V &visitor) noexcept; + simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -12153,7 +12923,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *peek() const noexcept; + simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -12161,56 +12931,56 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *advance() noexcept; + simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - simdjson_really_inline size_t remaining_len() const noexcept; + simdjson_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - simdjson_really_inline bool at_eof() const noexcept; + simdjson_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - simdjson_really_inline bool at_beginning() const noexcept; - simdjson_really_inline uint8_t last_structural() const noexcept; + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_value(const char *type) const noexcept; + simdjson_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_start_value(const char *type) const noexcept; + simdjson_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_end_value(const char *type) const noexcept; + simdjson_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_error(const char *error) const noexcept; + simdjson_inline void log_error(const char *error) const noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -12335,52 +13105,52 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_docum } // walk_document() -simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -simdjson_really_inline bool json_iterator::at_eof() const noexcept { +simdjson_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -simdjson_really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -12396,7 +13166,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root } } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -12428,18 +13198,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - simdjson_really_inline void append_s64(int64_t value) noexcept; + simdjson_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - simdjson_really_inline void append_u64(uint64_t value) noexcept; + simdjson_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - simdjson_really_inline void append_double(double value) noexcept; + simdjson_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -12447,24 +13217,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - simdjson_really_inline void skip() noexcept; + simdjson_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - simdjson_really_inline void skip_large_integer() noexcept; + simdjson_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - simdjson_really_inline void skip_double() noexcept; + simdjson_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -12472,50 +13242,50 @@ struct tape_writer { * all 64 bits, such as double and uint64_t. */ template - simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -simdjson_really_inline void tape_writer::append_double(double value) noexcept { +simdjson_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -simdjson_really_inline void tape_writer::skip() noexcept { +simdjson_inline void tape_writer::skip() noexcept { next_tape_loc++; } -simdjson_really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::skip_double() noexcept { +simdjson_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -12532,40 +13302,40 @@ namespace stage2 { struct tape_builder { template - simdjson_warn_unused static simdjson_really_inline error_code parse_document( + simdjson_warn_unused static simdjson_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - simdjson_warn_unused simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -12574,22 +13344,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -12597,18 +13367,18 @@ struct tape_builder { /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_really_inline tape_builder(dom::document &doc) noexcept; + simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_really_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_inline void start_container(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_document( +simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -12617,56 +13387,56 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_docum return iter.walk_document(builder); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value+1, dst); @@ -12678,16 +13448,16 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_strin return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -12709,42 +13479,42 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ return error; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -12753,24 +13523,24 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ // private: -simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -12783,13 +13553,13 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_contain return SUCCESS; } -simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -12815,7 +13585,7 @@ namespace ppc64 { namespace { namespace stage1 { -simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { // On PPC, we don't short-circuit this if there are no backslashes, because the branch gives us no // benefit and therefore makes things worse. // if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } @@ -12847,6 +13617,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { + return ppc64::stringparsing::parse_string(src, dst); +} + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } @@ -12910,17 +13684,17 @@ namespace { using namespace simd; struct json_character_block { - static simdjson_really_inline json_character_block classify(const simd::simd8x64& in); + static simdjson_inline json_character_block classify(const simd::simd8x64& in); - simdjson_really_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_really_inline uint64_t op() const noexcept { return _op; } - simdjson_really_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } + simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } + simdjson_inline uint64_t op() const noexcept { return _op; } + simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } uint64_t _whitespace; uint64_t _op; }; -simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { +simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why // we can't use the generic lookup_16. auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); @@ -12979,21 +13753,21 @@ simdjson_really_inline json_character_block json_character_block::classify(const return { whitespace, op }; } -simdjson_really_inline bool is_ascii(const simd8x64& input) { +simdjson_inline bool is_ascii(const simd8x64& input) { return input.reduce_or().is_ascii(); } -simdjson_unused simdjson_really_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 +simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0xc0u-1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); } -simdjson_really_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 +simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u-1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u-1); // Only 1111____ will be > 0 // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. return simd8(is_third_byte | is_fourth_byte) > int8_t(0); } @@ -13010,7 +13784,7 @@ namespace utf8_validation { using namespace simd; - simdjson_really_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) // Bit 1 = Too Long (ASCII followed by continuation) // Bit 2 = Overlong 3-byte @@ -13100,7 +13874,7 @@ using namespace simd; ); return (byte_1_high & byte_1_low & byte_2_high); } - simdjson_really_inline simd8 check_multibyte_lengths(const simd8 input, + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); @@ -13113,7 +13887,7 @@ using namespace simd; // Return nonzero if there are incomplete multibyte characters at the end of the block: // e.g. if there is a 4-byte character, but it's 3 bytes from the end. // - simdjson_really_inline simd8 is_incomplete(const simd8 input) { + simdjson_inline simd8 is_incomplete(const simd8 input) { // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): // ... 1111____ 111_____ 11______ #if SIMDJSON_IMPLEMENTATION_ICELAKE @@ -13125,14 +13899,14 @@ using namespace simd; 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #else static const uint8_t max_array[32] = { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1 + 255, 255, 255, 255, 255, 0xf0u-1, 0xe0u-1, 0xc0u-1 }; #endif const simd8 max_value(&max_array[sizeof(max_array)-sizeof(simd8)]); @@ -13150,7 +13924,7 @@ using namespace simd; // // Check whether the current bytes are valid UTF-8. // - simdjson_really_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) simd8 prev1 = input.prev<1>(prev_input); @@ -13161,13 +13935,21 @@ using namespace simd; // The only problem that can happen at EOF is that a multibyte character is too short // or a byte value too large in the last bytes: check_special_cases only checks for bytes // too large in the first of two bytes. - simdjson_really_inline void check_eof() { + simdjson_inline void check_eof() { // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't // possibly finish them. this->error |= this->prev_incomplete; } - simdjson_really_inline void check_next_input(const simd8x64& input) { +#ifndef SIMDJSON_IF_CONSTEXPR +#if SIMDJSON_CPLUSPLUS17 +#define SIMDJSON_IF_CONSTEXPR if constexpr +#else +#define SIMDJSON_IF_CONSTEXPR if +#endif +#endif + + simdjson_inline void check_next_input(const simd8x64& input) { if(simdjson_likely(is_ascii(input))) { this->error |= this->prev_incomplete; } else { @@ -13176,12 +13958,12 @@ using namespace simd; ||(simd8x64::NUM_CHUNKS == 2) || (simd8x64::NUM_CHUNKS == 4), "We support one, two or four chunks per 64-byte block."); - if(simd8x64::NUM_CHUNKS == 1) { + SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 1) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - } if(simd8x64::NUM_CHUNKS == 2) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 2) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } else if(simd8x64::NUM_CHUNKS == 4) { + } else SIMDJSON_IF_CONSTEXPR (simd8x64::NUM_CHUNKS == 4) { this->check_utf8_bytes(input.chunks[0], this->prev_input_block); this->check_utf8_bytes(input.chunks[1], input.chunks[0]); this->check_utf8_bytes(input.chunks[2], input.chunks[1]); @@ -13192,7 +13974,7 @@ using namespace simd; } } // do not forget to call check_eof! - simdjson_really_inline error_code errors() { + simdjson_inline error_code errors() { return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; } @@ -13220,10 +14002,10 @@ namespace { template struct buf_block_reader { public: - simdjson_really_inline buf_block_reader(const uint8_t *_buf, size_t _len); - simdjson_really_inline size_t block_index(); - simdjson_really_inline bool has_full_block() const; - simdjson_really_inline const uint8_t *full_block() const; + simdjson_inline buf_block_reader(const uint8_t *_buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t *full_block() const; /** * Get the last block, padded with spaces. * @@ -13233,8 +14015,8 @@ struct buf_block_reader { * * @return the number of effective characters in the last block. */ - simdjson_really_inline size_t get_remainder(uint8_t *dst) const; - simdjson_really_inline void advance(); + simdjson_inline size_t get_remainder(uint8_t *dst) const; + simdjson_inline void advance(); private: const uint8_t *buf; const size_t len; @@ -13273,23 +14055,23 @@ simdjson_unused static char * format_mask(uint64_t mask) { } template -simdjson_really_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} +simdjson_inline buf_block_reader::buf_block_reader(const uint8_t *_buf, size_t _len) : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {} template -simdjson_really_inline size_t buf_block_reader::block_index() { return idx; } +simdjson_inline size_t buf_block_reader::block_index() { return idx; } template -simdjson_really_inline bool buf_block_reader::has_full_block() const { +simdjson_inline bool buf_block_reader::has_full_block() const { return idx < lenminusstep; } template -simdjson_really_inline const uint8_t *buf_block_reader::full_block() const { +simdjson_inline const uint8_t *buf_block_reader::full_block() const { return &buf[idx]; } template -simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { +simdjson_inline size_t buf_block_reader::get_remainder(uint8_t *dst) const { if(len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. std::memcpy(dst, buf + idx, len - idx); @@ -13297,7 +14079,7 @@ simdjson_really_inline size_t buf_block_reader::get_remainder(uint8_t } template -simdjson_really_inline void buf_block_reader::advance() { +simdjson_inline void buf_block_reader::advance() { idx += STEP_SIZE; } @@ -13313,27 +14095,27 @@ namespace stage1 { struct json_string_block { // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : + simdjson_inline json_string_block(uint64_t backslash, uint64_t escaped, uint64_t quote, uint64_t in_string) : _backslash(backslash), _escaped(escaped), _quote(quote), _in_string(in_string) {} // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } + simdjson_inline uint64_t escaped() const { return _escaped; } // Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \) - simdjson_really_inline uint64_t escape() const { return _backslash & ~_escaped; } + simdjson_inline uint64_t escape() const { return _backslash & ~_escaped; } // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } + simdjson_inline uint64_t quote() const { return _quote; } // Start quotes of strings - simdjson_really_inline uint64_t string_start() const { return _quote & _in_string; } + simdjson_inline uint64_t string_start() const { return _quote & _in_string; } // End quotes of strings - simdjson_really_inline uint64_t string_end() const { return _quote & ~_in_string; } + simdjson_inline uint64_t string_end() const { return _quote & ~_in_string; } // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + simdjson_inline uint64_t string_content() const { return _in_string & ~_quote; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + simdjson_inline uint64_t string_tail() const { return _in_string ^ _quote; } // backslash characters uint64_t _backslash; @@ -13348,14 +14130,14 @@ struct json_string_block { // Scans blocks for string characters, storing the state necessary to do so class json_string_scanner { public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); + simdjson_inline json_string_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Intended to be defined by the implementation - simdjson_really_inline uint64_t find_escaped(uint64_t escape); - simdjson_really_inline uint64_t find_escaped_branchless(uint64_t escape); + simdjson_inline uint64_t find_escaped(uint64_t escape); + simdjson_inline uint64_t find_escaped_branchless(uint64_t escape); // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). uint64_t prev_in_string = 0ULL; @@ -13390,7 +14172,7 @@ class json_string_scanner { // desired | x | x x x x x x x x | // text | \\\ | \\\"\\\" \\\" \\"\\" | // -simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped_branchless(uint64_t backslash) { // If there was overflow, pretend the first character isn't a backslash backslash &= ~prev_escaped; uint64_t follows_escape = backslash << 1 | prev_escaped; @@ -13415,7 +14197,7 @@ simdjson_really_inline uint64_t json_string_scanner::find_escaped_branchless(uin // // Backslash sequences outside of quotes will be detected in stage 2. // -simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { const uint64_t backslash = in.eq('\\'); const uint64_t escaped = find_escaped(backslash); const uint64_t quote = in.eq('"') & ~escaped; @@ -13448,7 +14230,7 @@ simdjson_really_inline json_string_block json_string_scanner::next(const simd::s ); } -simdjson_really_inline error_code json_string_scanner::finish() { +simdjson_inline error_code json_string_scanner::finish() { if (prev_in_string) { return UNCLOSED_STRING; } @@ -13486,25 +14268,25 @@ namespace stage1 { struct json_block { public: // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_really_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} /** * The start of structurals. * In simdjson prior to v0.3, these were called the pseudo-structural characters. **/ - simdjson_really_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } /** All JSON whitespace (i.e. not in a string) */ - simdjson_really_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } // Helpers /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } // string and escape characters json_string_block _string; @@ -13519,12 +14301,12 @@ struct json_block { * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". * They may reside inside a string. **/ - simdjson_really_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } /** * The start of non-operator runs, like 123, true and "abc". * It main reside inside a string. **/ - simdjson_really_inline uint64_t potential_scalar_start() const noexcept { + simdjson_inline uint64_t potential_scalar_start() const noexcept { // The term "scalar" refers to anything except structural characters and white space // (so letters, numbers, quotes). // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space @@ -13535,7 +14317,7 @@ struct json_block { * Whether the given character is immediately after a non-operator like 123, true. * The characters following a quote are not included. */ - simdjson_really_inline uint64_t follows_potential_scalar() const noexcept { + simdjson_inline uint64_t follows_potential_scalar() const noexcept { // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a // white space. @@ -13559,10 +14341,10 @@ struct json_block { */ class json_scanner { public: - json_scanner() {} - simdjson_really_inline json_block next(const simd::simd8x64& in); + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); + simdjson_inline error_code finish(); private: // Whether the last character of the previous iteration is part of a scalar token @@ -13579,13 +14361,13 @@ class json_scanner { // // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); // -simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { +simdjson_inline uint64_t follows(const uint64_t match, uint64_t &overflow) { const uint64_t result = match << 1 | overflow; overflow = match >> 63; return result; } -simdjson_really_inline json_block json_scanner::next(const simd::simd8x64& in) { +simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { json_string_block strings = string_scanner.next(in); // identifies the white-space and the structural characters json_character_block characters = json_character_block::classify(in); @@ -13610,7 +14392,7 @@ simdjson_really_inline json_block json_scanner::next(const simd::simd8x64 - simdjson_really_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_really_inline error_code finish(uint8_t *dst_start, size_t &dst_len); + simdjson_inline void step(const uint8_t *block_buf, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t *dst_start, size_t &dst_len); json_scanner scanner{}; uint8_t *dst; }; -simdjson_really_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { +simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { uint64_t mask = block.whitespace(); dst += in.compress(mask, dst); } -simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { +simdjson_inline error_code json_minifier::finish(uint8_t *dst_start, size_t &dst_len) { error_code error = scanner.finish(); if (error) { dst_len = 0; return error; } dst_len = dst - dst_start; @@ -13660,7 +14442,7 @@ simdjson_really_inline error_code json_minifier::finish(uint8_t *dst_start, size } template<> -simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_minifier::step<128>(const uint8_t *block_buf, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block_buf); simd::simd8x64 in_2(block_buf+64); json_block block_1 = scanner.next(in_1); @@ -13671,7 +14453,7 @@ simdjson_really_inline void json_minifier::step<128>(const uint8_t *block_buf, b } template<> -simdjson_really_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_minifier::step<64>(const uint8_t *block_buf, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block_buf); json_block block_1 = scanner.next(in_1); this->next(block_buf, block_1); @@ -13744,7 +14526,7 @@ namespace { * complete document, therefore the last json buffer location is the end of the * batch. */ -simdjson_really_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { +simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &parser) { // Variant: do not count separately, just figure out depth if(parser.n_structural_indexes == 0) { return 0; } auto arr_cnt = 0; @@ -13819,7 +14601,7 @@ class bit_indexer { public: uint32_t *tail; - simdjson_really_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} + simdjson_inline bit_indexer(uint32_t *index_buf) : tail(index_buf) {} // flatten out values in 'bits' assuming that they are are to have values of idx // plus their position in the bitvector, and store these indexes at @@ -13830,9 +14612,9 @@ class bit_indexer { // If the kernel sets SIMDJSON_CUSTOM_BIT_INDEXER, then it will provide its own // version of the code. #ifdef SIMDJSON_CUSTOM_BIT_INDEXER - simdjson_really_inline void write(uint32_t idx, uint64_t bits); + simdjson_inline void write(uint32_t idx, uint64_t bits); #else - simdjson_really_inline void write(uint32_t idx, uint64_t bits) { + simdjson_inline void write(uint32_t idx, uint64_t bits) { // In some instances, the next branch is expensive because it is mispredicted. // Unfortunately, in other cases, // it helps tremendously. @@ -13941,11 +14723,11 @@ class json_structural_indexer { static error_code index(const uint8_t *buf, size_t len, dom_parser_implementation &parser, stage1_mode partial) noexcept; private: - simdjson_really_inline json_structural_indexer(uint32_t *structural_indexes); + simdjson_inline json_structural_indexer(uint32_t *structural_indexes); template - simdjson_really_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; - simdjson_really_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_really_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); + simdjson_inline void step(const uint8_t *block, buf_block_reader &reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation &parser, size_t idx, size_t len, stage1_mode partial); json_scanner scanner{}; utf8_checker checker{}; @@ -13954,26 +14736,26 @@ class json_structural_indexer { uint64_t unescaped_chars_error = 0; }; -simdjson_really_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} +simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t *structural_indexes) : indexer{structural_indexes} {} // Skip the last character if it is partial -simdjson_really_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { +simdjson_inline size_t trim_partial_utf8(const uint8_t *buf, size_t len) { if (simdjson_unlikely(len < 3)) { switch (len) { case 2: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 2 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 2 bytes left return len; case 1: - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left return len; case 0: return len; } } - if (buf[len-1] >= 0b11000000) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len-2] >= 0b11100000) { return len-2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len-3] >= 0b11110000) { return len-3; } // 4-byte characters with only 3 bytes left + if (buf[len-1] >= 0xc0) { return len-1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len-2] >= 0xe0) { return len-2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len-3] >= 0xf0) { return len-3; } // 4-byte characters with only 3 bytes left return len; } @@ -14022,7 +14804,7 @@ error_code json_structural_indexer::index(const uint8_t *buf, size_t len, dom_pa } template<> -simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<128>(const uint8_t *block, buf_block_reader<128> &reader) noexcept { simd::simd8x64 in_1(block); simd::simd8x64 in_2(block+64); json_block block_1 = scanner.next(in_1); @@ -14033,14 +14815,14 @@ simdjson_really_inline void json_structural_indexer::step<128>(const uint8_t *bl } template<> -simdjson_really_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { +simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf_block_reader<64> &reader) noexcept { simd::simd8x64 in_1(block); json_block block_1 = scanner.next(in_1); this->next(in_1, block_1, reader.block_index()); reader.advance(); } -simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { +simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { uint64_t unescaped = in.lteq(0x1F); checker.check_next_input(in); indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser @@ -14048,7 +14830,7 @@ simdjson_really_inline void json_structural_indexer::next(const simd::simd8x64 backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab +// u not handled in this table as it's complex +static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +// handle a unicode codepoint +// write appropriate values into dest +// src will advance 6 bytes or 12 bytes +// dest will advance a variable amount (return via pointer) +// return true if the unicode codepoint was valid +// We work in little-endian then swap at write time +simdjson_warn_unused +simdjson_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, + uint8_t **dst_ptr) { + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion isn't valid; we defer the check for this to inside the + // multilingual plane check + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t *src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + return false; + } + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + return false; + } + + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + return false; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; +} + +/** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_nullptrptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst)) { + return nullptr; + } + } else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + /* can't be reached */ + return nullptr; +} + +} // namespace stringparsing +} // unnamed namespace +} // namespace westmere +} // namespace simdjson +/* end file src/generic/stage2/stringparsing.h */ /* begin file src/generic/stage2/tape_builder.h */ /* begin file src/generic/stage2/json_iterator.h */ /* begin file src/generic/stage2/logger.h */ @@ -14219,7 +15146,7 @@ namespace logger { static int log_depth; // Not threadsafe. Log only. // Helper to turn unprintable or newline characters into spaces - static simdjson_really_inline char printable_char(char c) { + static simdjson_inline char printable_char(char c) { if (c >= 0x20) { return c; } else { @@ -14228,7 +15155,7 @@ namespace logger { } // Print the header and set up log_start - static simdjson_really_inline void log_start() { + static simdjson_inline void log_start() { if (LOG_ENABLED) { log_depth = 0; printf("\n"); @@ -14237,7 +15164,7 @@ namespace logger { } } - simdjson_unused static simdjson_really_inline void log_string(const char *message) { + simdjson_unused static simdjson_inline void log_string(const char *message) { if (LOG_ENABLED) { printf("%s\n", message); } @@ -14245,7 +15172,7 @@ namespace logger { // Logs a single line from the stage 2 DOM parser template - static simdjson_really_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { + static simdjson_inline void log_line(S &structurals, const char *title_prefix, const char *title, const char *detail) { if (LOG_ENABLED) { printf("| %*s%s%-*s ", log_depth*2, "", title_prefix, LOG_EVENT_LEN - log_depth*2 - int(strlen(title_prefix)), title); auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural-1; @@ -14323,14 +15250,14 @@ class json_iterator { * - increment_count(iter) - each time a value is found in an array or object. */ template - simdjson_warn_unused simdjson_really_inline error_code walk_document(V &visitor) noexcept; + simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** * Create an iterator capable of walking a JSON document. * * The document must have already passed through stage 1. */ - simdjson_really_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); + simdjson_inline json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index); /** * Look at the next token. @@ -14339,7 +15266,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *peek() const noexcept; + simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. * @@ -14347,56 +15274,56 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ - simdjson_really_inline const uint8_t *advance() noexcept; + simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. */ - simdjson_really_inline size_t remaining_len() const noexcept; + simdjson_inline size_t remaining_len() const noexcept; /** * Check if we are at the end of the document. * * If this is true, there are no more tokens. */ - simdjson_really_inline bool at_eof() const noexcept; + simdjson_inline bool at_eof() const noexcept; /** * Check if we are at the beginning of the document. */ - simdjson_really_inline bool at_beginning() const noexcept; - simdjson_really_inline uint8_t last_structural() const noexcept; + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; /** * Log that a value has been found. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_value(const char *type) const noexcept; + simdjson_inline void log_value(const char *type) const noexcept; /** * Log the start of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_start_value(const char *type) const noexcept; + simdjson_inline void log_start_value(const char *type) const noexcept; /** * Log the end of a multipart value. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_end_value(const char *type) const noexcept; + simdjson_inline void log_end_value(const char *type) const noexcept; /** * Log an error. * * Set LOG_ENABLED=true in logger.h to see logging. */ - simdjson_really_inline void log_error(const char *error) const noexcept; + simdjson_inline void log_error(const char *error) const noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V &visitor, const uint8_t *value) noexcept; template - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_document(V &visitor) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); // @@ -14521,52 +15448,52 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::walk_docum } // walk_document() -simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) +simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) : buf{_dom_parser.buf}, next_structural{&_dom_parser.structural_indexes[start_structural_index]}, dom_parser{_dom_parser} { } -simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept { +simdjson_inline const uint8_t *json_iterator::peek() const noexcept { return &buf[*(next_structural)]; } -simdjson_really_inline const uint8_t *json_iterator::advance() noexcept { +simdjson_inline const uint8_t *json_iterator::advance() noexcept { return &buf[*(next_structural++)]; } -simdjson_really_inline size_t json_iterator::remaining_len() const noexcept { +simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); } -simdjson_really_inline bool json_iterator::at_eof() const noexcept { +simdjson_inline bool json_iterator::at_eof() const noexcept { return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; } -simdjson_really_inline bool json_iterator::at_beginning() const noexcept { +simdjson_inline bool json_iterator::at_beginning() const noexcept { return next_structural == dom_parser.structural_indexes.get(); } -simdjson_really_inline uint8_t json_iterator::last_structural() const noexcept { +simdjson_inline uint8_t json_iterator::last_structural() const noexcept { return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; } -simdjson_really_inline void json_iterator::log_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_value(const char *type) const noexcept { logger::log_line(*this, "", type, ""); } -simdjson_really_inline void json_iterator::log_start_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_start_value(const char *type) const noexcept { logger::log_line(*this, "+", type, ""); if (logger::LOG_ENABLED) { logger::log_depth++; } } -simdjson_really_inline void json_iterator::log_end_value(const char *type) const noexcept { +simdjson_inline void json_iterator::log_end_value(const char *type) const noexcept { if (logger::LOG_ENABLED) { logger::log_depth--; } logger::log_line(*this, "-", type, ""); } -simdjson_really_inline void json_iterator::log_error(const char *error) const noexcept { +simdjson_inline void json_iterator::log_error(const char *error) const noexcept { logger::log_line(*this, "", "ERROR", error); } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); @@ -14582,7 +15509,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_root } } template -simdjson_warn_unused simdjson_really_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V &visitor, const uint8_t *value) noexcept { switch (*value) { case '"': return visitor.visit_string(*this, value); case 't': return visitor.visit_true_atom(*this, value); @@ -14614,18 +15541,18 @@ struct tape_writer { uint64_t *next_tape_loc; /** Write a signed 64-bit value to tape. */ - simdjson_really_inline void append_s64(int64_t value) noexcept; + simdjson_inline void append_s64(int64_t value) noexcept; /** Write an unsigned 64-bit value to tape. */ - simdjson_really_inline void append_u64(uint64_t value) noexcept; + simdjson_inline void append_u64(uint64_t value) noexcept; /** Write a double value to tape. */ - simdjson_really_inline void append_double(double value) noexcept; + simdjson_inline void append_double(double value) noexcept; /** * Append a tape entry (an 8-bit type,and 56 bits worth of value). */ - simdjson_really_inline void append(uint64_t val, internal::tape_type t) noexcept; + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; /** * Skip the current tape entry without writing. @@ -14633,24 +15560,24 @@ struct tape_writer { * Used to skip the start of the container, since we'll come back later to fill it in when the * container ends. */ - simdjson_really_inline void skip() noexcept; + simdjson_inline void skip() noexcept; /** * Skip the number of tape entries necessary to write a large u64 or i64. */ - simdjson_really_inline void skip_large_integer() noexcept; + simdjson_inline void skip_large_integer() noexcept; /** * Skip the number of tape entries necessary to write a double. */ - simdjson_really_inline void skip_double() noexcept; + simdjson_inline void skip_double() noexcept; /** * Write a value to a known location on tape. * * Used to go back and write out the start of a container after the container ends. */ - simdjson_really_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; + simdjson_inline static void write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept; private: /** @@ -14658,50 +15585,50 @@ struct tape_writer { * all 64 bits, such as double and uint64_t. */ template - simdjson_really_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; }; // struct number_writer -simdjson_really_inline void tape_writer::append_s64(int64_t value) noexcept { +simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { append2(0, value, internal::tape_type::INT64); } -simdjson_really_inline void tape_writer::append_u64(uint64_t value) noexcept { +simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { append(0, internal::tape_type::UINT64); *next_tape_loc = value; next_tape_loc++; } /** Write a double value to tape. */ -simdjson_really_inline void tape_writer::append_double(double value) noexcept { +simdjson_inline void tape_writer::append_double(double value) noexcept { append2(0, value, internal::tape_type::DOUBLE); } -simdjson_really_inline void tape_writer::skip() noexcept { +simdjson_inline void tape_writer::skip() noexcept { next_tape_loc++; } -simdjson_really_inline void tape_writer::skip_large_integer() noexcept { +simdjson_inline void tape_writer::skip_large_integer() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::skip_double() noexcept { +simdjson_inline void tape_writer::skip_double() noexcept { next_tape_loc += 2; } -simdjson_really_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { *next_tape_loc = val | ((uint64_t(char(t))) << 56); next_tape_loc++; } template -simdjson_really_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { append(val, t); static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); memcpy(next_tape_loc, &val2, sizeof(val2)); next_tape_loc++; } -simdjson_really_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { +simdjson_inline void tape_writer::write(uint64_t &tape_loc, uint64_t val, internal::tape_type t) noexcept { tape_loc = val | ((uint64_t(char(t))) << 56); } @@ -14718,40 +15645,40 @@ namespace stage2 { struct tape_builder { template - simdjson_warn_unused static simdjson_really_inline error_code parse_document( + simdjson_warn_unused static simdjson_inline error_code parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept; /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_really_inline error_code visit_document_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator &iter) noexcept; /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator &iter) noexcept; /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_array_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator &iter) noexcept; /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_array(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator &iter) noexcept; /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_start(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator &iter) noexcept; /** * Called when a key in a field is encountered. * * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array * will be called after this with the field value. */ - simdjson_warn_unused simdjson_really_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator &iter, const uint8_t *key) noexcept; /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_really_inline error_code visit_object_end(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator &iter) noexcept; /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_empty_object(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator &iter) noexcept; /** * Called when a string, number, boolean or null is found. */ - simdjson_warn_unused simdjson_really_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator &iter, const uint8_t *value) noexcept; /** * Called when a string, number, boolean or null is found at the top level of a document (i.e. * when there is no array or object and the entire document is a single string, number, boolean or @@ -14760,22 +15687,22 @@ struct tape_builder { * This is separate from primitive() because simdjson's normal primitive parsing routines assume * there is at least one more token after the value, which is only true in an array or object. */ - simdjson_warn_unused simdjson_really_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator &iter, const uint8_t *value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; - simdjson_warn_unused simdjson_really_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_really_inline error_code increment_count(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; @@ -14783,18 +15710,18 @@ struct tape_builder { /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_really_inline tape_builder(dom::document &doc) noexcept; + simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_really_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; - simdjson_really_inline void start_container(json_iterator &iter) noexcept; - simdjson_warn_unused simdjson_really_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_really_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_really_inline uint8_t *on_start_string(json_iterator &iter) noexcept; - simdjson_really_inline void on_end_string(uint8_t *dst) noexcept; + simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; + simdjson_inline void start_container(json_iterator &iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; + simdjson_inline void on_end_string(uint8_t *dst) noexcept; }; // class tape_builder template -simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_document( +simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( dom_parser_implementation &dom_parser, dom::document &doc) noexcept { dom_parser.doc = &doc; @@ -14803,56 +15730,56 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::parse_docum return iter.walk_document(builder); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); dst = stringparsing::parse_string(value+1, dst); @@ -14864,16 +15791,16 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_strin return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); return numberparsing::parse_number(value, tape); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -14895,42 +15822,42 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ return error; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); @@ -14939,24 +15866,24 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_ // private: -simdjson_really_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_really_inline void tape_builder::start_container(json_iterator &iter) noexcept { +simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -14969,13 +15896,13 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::end_contain return SUCCESS; } -simdjson_really_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_really_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -15002,7 +15929,7 @@ namespace westmere { namespace { namespace stage1 { -simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { +simdjson_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) { if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; } return find_escaped_branchless(backslash); } @@ -15032,6 +15959,10 @@ simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::docu return stage2::tape_builder::parse_document(*this, _doc); } +simdjson_warn_unused uint8_t *dom_parser_implementation::parse_string(const uint8_t *src, uint8_t *dst) const noexcept { + return westmere::stringparsing::parse_string(src, dst); +} + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept { auto error = stage1(_buf, _len, stage1_mode::regular); if (error) { return error; } diff --git a/simdjson/simdjson.h b/simdjson/simdjson.h index 67cb53b..f9d8081 100644 --- a/simdjson/simdjson.h +++ b/simdjson/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2022-06-02 13:56:20 -0400. Do not edit! */ +/* auto-generated on 2022-11-23 10:31:42 -0500. Do not edit! */ /* begin file include/simdjson.h */ #ifndef SIMDJSON_H #define SIMDJSON_H @@ -43,14 +43,14 @@ #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION 2.0.3 +#define SIMDJSON_VERSION 3.0.1 namespace simdjson { enum { /** * The major version (MAJOR.minor.revision) of simdjson being used. */ - SIMDJSON_VERSION_MAJOR = 2, + SIMDJSON_VERSION_MAJOR = 3, /** * The minor version (major.MINOR.revision) of simdjson being used. */ @@ -58,7 +58,7 @@ enum { /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 3 + SIMDJSON_VERSION_REVISION = 1 }; } // namespace simdjson @@ -329,12 +329,12 @@ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; /** * The amount of padding needed in a buffer to parse JSON. * - * the input buf should be readable up to buf + SIMDJSON_PADDING + * The input buf should be readable up to buf + SIMDJSON_PADDING * this is a stopgap; there should be a better description of the * main loop and its behavior that abstracts over this * See https://github.com/simdjson/simdjson/issues/174 */ -constexpr size_t SIMDJSON_PADDING = 32; +constexpr size_t SIMDJSON_PADDING = 64; /** * By default, simdjson supports this many nested objects and arrays. @@ -460,6 +460,18 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024; #endif // MSC_VER +#if defined(simdjson_inline) + // Prefer the user's definition of simdjson_inline; don't define it ourselves. +#elif defined(__GNUC__) && !defined(__OPTIMIZE__) + // If optimizations are disabled, forcing inlining can lead to significant + // code bloat and high compile times. Don't use simdjson_really_inline for + // unoptimized builds. + #define simdjson_inline inline +#else + // Force inlining for most simdjson functions. + #define simdjson_inline simdjson_really_inline +#endif + #if defined(SIMDJSON_VISUAL_STUDIO) /** * Windows users need to do some extra work when building @@ -2197,11 +2209,30 @@ namespace std { /// If EXPR is an error, returns it. #define SIMDJSON_TRY(EXPR) { auto _err = (EXPR); if (_err) { return _err; } } +// Unless the programmer has already set SIMDJSON_DEVELOPMENT_CHECKS, +// we want to set it under debug builds. We detect a debug build +// under Visual Studio when the _DEBUG macro is set. Under the other +// compilers, we use the fact that they define __OPTIMIZE__ whenever +// they allow optimizations. +// It is possible that this could miss some cases where SIMDJSON_DEVELOPMENT_CHECKS +// is helpful, but the programmer can set the macro SIMDJSON_DEVELOPMENT_CHECKS. +// It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer +// sets _DEBUG in a release build under Visual Studio, or if some compiler fails to +// set the __OPTIMIZE__ macro). #ifndef SIMDJSON_DEVELOPMENT_CHECKS -#ifndef NDEBUG -#define SIMDJSON_DEVELOPMENT_CHECKS -#endif -#endif +#ifdef _MSC_VER +// Visual Studio seems to set _DEBUG for debug builds. +#ifdef _DEBUG +#define SIMDJSON_DEVELOPMENT_CHECKS 1 +#endif // _DEBUG +#else // _MSC_VER +// All other compilers appear to set __OPTIMIZE__ to a positive integer +// when the compiler is optimizing. +#ifndef __OPTIMIZE__ +#define SIMDJSON_DEVELOPMENT_CHECKS 1 +#endif // __OPTIMIZE__ +#endif // _MSC_VER +#endif // SIMDJSON_DEVELOPMENT_CHECKS // The SIMDJSON_CHECK_EOF macro is a feature flag for the "don't require padding" // feature. @@ -2222,6 +2253,13 @@ namespace std { # define simdjson_fallthrough do {} while (0) /* fallthrough */ #endif // simdjson_fallthrough + +#if SIMDJSON_DEVELOPMENT_CHECKS +#define SIMDJSON_DEVELOPMENT_ASSERT(expr) do { assert ((expr)); } while (0) +#else +#define SIMDJSON_DEVELOPMENT_ASSERT(expr) do { } while (0) +#endif + #endif // SIMDJSON_COMMON_DEFS_H /* end file include/simdjson/common_defs.h */ @@ -2249,7 +2287,7 @@ enum error_code { SUCCESS = 0, ///< No error CAPACITY, ///< This parser can't support a document that big MEMALLOC, ///< Error allocating memory, most likely out of memory - TAPE_ERROR, ///< Something went wrong while writing to the tape (stage 2), this is a generic error + TAPE_ERROR, ///< Something went wrong, this is a generic error DEPTH_ERROR, ///< Your document exceeds the user-specified depth limitation STRING_ERROR, ///< Problem while parsing a string T_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 't' @@ -2276,6 +2314,7 @@ enum error_code { INCOMPLETE_ARRAY_OR_OBJECT, ///< The document ends early. SCALAR_DOCUMENT_AS_VALUE, ///< A scalar document is treated as a value. OUT_OF_BOUNDS, ///< Attempted to access location outside of document. + TRAILING_CONTENT, ///< Unexpected trailing content in the JSON input NUM_ERROR_CODES }; @@ -2342,22 +2381,22 @@ struct simdjson_result_base : protected std::pair { /** * Create a new empty result with error = UNINITIALIZED. */ - simdjson_really_inline simdjson_result_base() noexcept; + simdjson_inline simdjson_result_base() noexcept; /** * Create a new error result. */ - simdjson_really_inline simdjson_result_base(error_code error) noexcept; + simdjson_inline simdjson_result_base(error_code error) noexcept; /** * Create a new successful result. */ - simdjson_really_inline simdjson_result_base(T &&value) noexcept; + simdjson_inline simdjson_result_base(T &&value) noexcept; /** * Create a new result with both things (use if you don't want to branch when creating the result). */ - simdjson_really_inline simdjson_result_base(T &&value, error_code error) noexcept; + simdjson_inline simdjson_result_base(T &&value, error_code error) noexcept; /** * Move the value and the error to the provided variables. @@ -2365,26 +2404,19 @@ struct simdjson_result_base : protected std::pair { * @param value The variable to assign the value to. May not be set if there is an error. * @param error The variable to assign the error to. Set to SUCCESS if there is no error. */ - simdjson_really_inline void tie(T &value, error_code &error) && noexcept; + simdjson_inline void tie(T &value, error_code &error) && noexcept; /** * Move the value to the provided variable. * * @param value The variable to assign the value to. May not be set if there is an error. */ - simdjson_really_inline error_code get(T &value) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_really_inline const T &value(error_code &error) const & noexcept; + simdjson_inline error_code get(T &value) && noexcept; /** * The error. */ - simdjson_really_inline error_code error() const noexcept; + simdjson_inline error_code error() const noexcept; #if SIMDJSON_EXCEPTIONS @@ -2393,41 +2425,41 @@ struct simdjson_result_base : protected std::pair { * * @throw simdjson_error if there was an error. */ - simdjson_really_inline T& value() & noexcept(false); + simdjson_inline T& value() & noexcept(false); /** * Take the result value (move it). * * @throw simdjson_error if there was an error. */ - simdjson_really_inline T&& value() && noexcept(false); + simdjson_inline T&& value() && noexcept(false); /** * Take the result value (move it). * * @throw simdjson_error if there was an error. */ - simdjson_really_inline T&& take_value() && noexcept(false); + simdjson_inline T&& take_value() && noexcept(false); /** * Cast to the value (will throw on error). * * @throw simdjson_error if there was an error. */ - simdjson_really_inline operator T&&() && noexcept(false); + simdjson_inline operator T&&() && noexcept(false); #endif // SIMDJSON_EXCEPTIONS /** * Get the result value. This function is safe if and only * the error() method returns a value that evaluates to false. */ - simdjson_really_inline const T& value_unsafe() const& noexcept; + simdjson_inline const T& value_unsafe() const& noexcept; /** * Take the result value (move it). This function is safe if and only * the error() method returns a value that evaluates to false. */ - simdjson_really_inline T&& value_unsafe() && noexcept; + simdjson_inline T&& value_unsafe() && noexcept; }; // struct simdjson_result_base @@ -2443,19 +2475,19 @@ struct simdjson_result : public internal::simdjson_result_base { /** * @private Create a new empty result with error = UNINITIALIZED. */ - simdjson_really_inline simdjson_result() noexcept; + simdjson_inline simdjson_result() noexcept; /** * @private Create a new error result. */ - simdjson_really_inline simdjson_result(T &&value) noexcept; + simdjson_inline simdjson_result(T &&value) noexcept; /** * @private Create a new successful result. */ - simdjson_really_inline simdjson_result(error_code error_code) noexcept; + simdjson_inline simdjson_result(error_code error_code) noexcept; /** * @private Create a new result with both things (use if you don't want to branch when creating the result). */ - simdjson_really_inline simdjson_result(T &&value, error_code error) noexcept; + simdjson_inline simdjson_result(T &&value, error_code error) noexcept; /** * Move the value and the error to the provided variables. @@ -2463,19 +2495,19 @@ struct simdjson_result : public internal::simdjson_result_base { * @param value The variable to assign the value to. May not be set if there is an error. * @param error The variable to assign the error to. Set to SUCCESS if there is no error. */ - simdjson_really_inline void tie(T &value, error_code &error) && noexcept; + simdjson_inline void tie(T &value, error_code &error) && noexcept; /** * Move the value to the provided variable. * * @param value The variable to assign the value to. May not be set if there is an error. */ - simdjson_warn_unused simdjson_really_inline error_code get(T &value) && noexcept; + simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept; /** * The error. */ - simdjson_really_inline error_code error() const noexcept; + simdjson_inline error_code error() const noexcept; #if SIMDJSON_EXCEPTIONS @@ -2484,41 +2516,41 @@ struct simdjson_result : public internal::simdjson_result_base { * * @throw simdjson_error if there was an error. */ - simdjson_really_inline T& value() & noexcept(false); + simdjson_inline T& value() & noexcept(false); /** * Take the result value (move it). * * @throw simdjson_error if there was an error. */ - simdjson_really_inline T&& value() && noexcept(false); + simdjson_inline T&& value() && noexcept(false); /** * Take the result value (move it). * * @throw simdjson_error if there was an error. */ - simdjson_really_inline T&& take_value() && noexcept(false); + simdjson_inline T&& take_value() && noexcept(false); /** * Cast to the value (will throw on error). * * @throw simdjson_error if there was an error. */ - simdjson_really_inline operator T&&() && noexcept(false); + simdjson_inline operator T&&() && noexcept(false); #endif // SIMDJSON_EXCEPTIONS /** * Get the result value. This function is safe if and only * the error() method returns a value that evaluates to false. */ - simdjson_really_inline const T& value_unsafe() const& noexcept; + simdjson_inline const T& value_unsafe() const& noexcept; /** * Take the result value (move it). This function is safe if and only * the error() method returns a value that evaluates to false. */ - simdjson_really_inline T&& value_unsafe() && noexcept; + simdjson_inline T&& value_unsafe() && noexcept; }; // struct simdjson_result @@ -2926,6 +2958,22 @@ class dom_parser_implementation { */ simdjson_warn_unused virtual error_code stage2_next(dom::document &doc) noexcept = 0; + /** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_nullptrptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + * + * Overridden by each implementation. + * + * @param str pointer to the beginning of a valid UTF-8 JSON string, must end with an unescaped quote. + * @param dst pointer to a destination buffer, it must point a region in memory of sufficient size. + * @return end of the of the written region (exclusive) or nullptr in case of error. + */ + simdjson_warn_unused virtual uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept = 0; + /** * Change the capacity of this parser. * @@ -2968,14 +3016,14 @@ class dom_parser_implementation { * * @return Current capacity, in bytes. */ - simdjson_really_inline size_t capacity() const noexcept; + simdjson_inline size_t capacity() const noexcept; /** * The maximum level of nested object and arrays supported by this parser. * * @return Maximum depth, in bytes. */ - simdjson_really_inline size_t max_depth() const noexcept; + simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -2987,6 +3035,7 @@ class dom_parser_implementation { */ simdjson_warn_unused inline error_code allocate(size_t capacity, size_t max_depth) noexcept; + protected: /** * The maximum document length this parser supports. @@ -3003,23 +3052,23 @@ class dom_parser_implementation { size_t _max_depth{0}; // Declaring these so that subclasses can use them to implement their constructors. - simdjson_really_inline dom_parser_implementation() noexcept; - simdjson_really_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; - simdjson_really_inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; + simdjson_inline dom_parser_implementation() noexcept; + simdjson_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept; + simdjson_inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept; - simdjson_really_inline dom_parser_implementation(const dom_parser_implementation &) noexcept = delete; - simdjson_really_inline dom_parser_implementation &operator=(const dom_parser_implementation &other) noexcept = delete; + simdjson_inline dom_parser_implementation(const dom_parser_implementation &) noexcept = delete; + simdjson_inline dom_parser_implementation &operator=(const dom_parser_implementation &other) noexcept = delete; }; // class dom_parser_implementation -simdjson_really_inline dom_parser_implementation::dom_parser_implementation() noexcept = default; -simdjson_really_inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; -simdjson_really_inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; +simdjson_inline dom_parser_implementation::dom_parser_implementation() noexcept = default; +simdjson_inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation &&other) noexcept = default; +simdjson_inline dom_parser_implementation &dom_parser_implementation::operator=(dom_parser_implementation &&other) noexcept = default; -simdjson_really_inline size_t dom_parser_implementation::capacity() const noexcept { +simdjson_inline size_t dom_parser_implementation::capacity() const noexcept { return _capacity; } -simdjson_really_inline size_t dom_parser_implementation::max_depth() const noexcept { +simdjson_inline size_t dom_parser_implementation::max_depth() const noexcept { return _max_depth; } @@ -3159,7 +3208,7 @@ constexpr uint32_t cpuid_avx512pf_bit = 1 << 26; ///< @private bit 26 of EBX constexpr uint32_t cpuid_avx512er_bit = 1 << 27; ///< @private bit 27 of EBX for EAX=0x7 constexpr uint32_t cpuid_avx512cd_bit = 1 << 28; ///< @private bit 28 of EBX for EAX=0x7 constexpr uint32_t cpuid_avx512bw_bit = 1 << 30; ///< @private bit 30 of EBX for EAX=0x7 -constexpr uint32_t cpuid_avx512vl_bit = 1 << 31; ///< @private bit 31 of EBX for EAX=0x7 +constexpr uint32_t cpuid_avx512vl_bit = 1U << 31; ///< @private bit 31 of EBX for EAX=0x7 constexpr uint32_t cpuid_avx512vbmi2_bit = 1 << 6; ///< @private bit 6 of ECX for EAX=0x7 constexpr uint32_t cpuid_sse42_bit = 1 << 20; ///< @private bit 20 of ECX for EAX=0x1 constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; ///< @private bit 1 of ECX for EAX=0x1 @@ -3287,15 +3336,13 @@ namespace simdjson { * @return true if the string is valid UTF-8. */ simdjson_warn_unused bool validate_utf8(const char * buf, size_t len) noexcept; - - /** * Validate the UTF-8 string. * * @param sv the string_view to validate. * @return true if the string is valid UTF-8. */ -simdjson_really_inline simdjson_warn_unused bool validate_utf8(const std::string_view sv) noexcept { +simdjson_inline simdjson_warn_unused bool validate_utf8(const std::string_view sv) noexcept { return validate_utf8(sv.data(), sv.size()); } @@ -3305,7 +3352,7 @@ simdjson_really_inline simdjson_warn_unused bool validate_utf8(const std::string * @param p the string to validate. * @return true if the string is valid UTF-8. */ -simdjson_really_inline simdjson_warn_unused bool validate_utf8(const std::string& s) noexcept { +simdjson_inline simdjson_warn_unused bool validate_utf8(const std::string& s) noexcept { return validate_utf8(s.data(), s.size()); } @@ -3328,7 +3375,7 @@ class implementation { * const implementation *impl = simdjson::get_active_implementation(); * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; * - * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + * @return the name of the implementation, e.g. "haswell", "westmere", "arm64". */ virtual const std::string &name() const { return _name; } @@ -3338,7 +3385,7 @@ class implementation { * const implementation *impl = simdjson::get_active_implementation(); * cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl; * - * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + * @return the description of the implementation, e.g. "Intel/AMD AVX2", "Intel/AMD SSE4.2", "ARM NEON". */ virtual const std::string &description() const { return _description; } @@ -3347,8 +3394,7 @@ class implementation { * and the current CPU match. This function may poll the current CPU/system * and should therefore not be called too often if performance is a concern. * - * - * @return true if the implementation can be safely used on the current system (determined at runtime) + * @return true if the implementation can be safely used on the current system (determined at runtime). */ bool supported_by_runtime_system() const; @@ -3357,7 +3403,7 @@ class implementation { * * The instruction sets this implementation is compiled against. * - * @return a mask of all required `internal::instruction_set::` values + * @return a mask of all required `internal::instruction_set::` values. */ virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; }; @@ -3370,7 +3416,7 @@ class implementation { * @param capacity The largest document that will be passed to the parser. * @param max_depth The maximum JSON object/array nesting this parser is expected to handle. * @param dst The place to put the resulting parser implementation. - * @return the name of the implementation, e.g. "haswell", "westmere", "arm64" + * @return the error code, or SUCCESS if there was no error. */ virtual error_code create_dom_parser_implementation( size_t capacity, @@ -3407,7 +3453,7 @@ class implementation { protected: /** @private Construct an implementation with the given name and description. For subclasses. */ - simdjson_really_inline implementation( + simdjson_inline implementation( std::string_view name, std::string_view description, uint32_t required_instruction_sets @@ -3445,7 +3491,7 @@ namespace internal { class available_implementation_list { public: /** Get the list of available implementations compiled into simdjson */ - simdjson_really_inline available_implementation_list() {} + simdjson_inline available_implementation_list() {} /** Number of implementations */ size_t size() const noexcept; /** STL const begin() iterator */ @@ -3572,7 +3618,7 @@ namespace internal { // template -simdjson_really_inline void simdjson_result_base::tie(T &value, error_code &error) && noexcept { +simdjson_inline void simdjson_result_base::tie(T &value, error_code &error) && noexcept { error = this->second; if (!error) { value = std::forward>(*this).first; @@ -3580,64 +3626,64 @@ simdjson_really_inline void simdjson_result_base::tie(T &value, error_code &e } template -simdjson_warn_unused simdjson_really_inline error_code simdjson_result_base::get(T &value) && noexcept { +simdjson_warn_unused simdjson_inline error_code simdjson_result_base::get(T &value) && noexcept { error_code error; std::forward>(*this).tie(value, error); return error; } template -simdjson_really_inline error_code simdjson_result_base::error() const noexcept { +simdjson_inline error_code simdjson_result_base::error() const noexcept { return this->second; } #if SIMDJSON_EXCEPTIONS template -simdjson_really_inline T& simdjson_result_base::value() & noexcept(false) { +simdjson_inline T& simdjson_result_base::value() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return this->first; } template -simdjson_really_inline T&& simdjson_result_base::value() && noexcept(false) { +simdjson_inline T&& simdjson_result_base::value() && noexcept(false) { return std::forward>(*this).take_value(); } template -simdjson_really_inline T&& simdjson_result_base::take_value() && noexcept(false) { +simdjson_inline T&& simdjson_result_base::take_value() && noexcept(false) { if (error()) { throw simdjson_error(error()); } return std::forward(this->first); } template -simdjson_really_inline simdjson_result_base::operator T&&() && noexcept(false) { +simdjson_inline simdjson_result_base::operator T&&() && noexcept(false) { return std::forward>(*this).take_value(); } #endif // SIMDJSON_EXCEPTIONS template -simdjson_really_inline const T& simdjson_result_base::value_unsafe() const& noexcept { +simdjson_inline const T& simdjson_result_base::value_unsafe() const& noexcept { return this->first; } template -simdjson_really_inline T&& simdjson_result_base::value_unsafe() && noexcept { +simdjson_inline T&& simdjson_result_base::value_unsafe() && noexcept { return std::forward(this->first); } template -simdjson_really_inline simdjson_result_base::simdjson_result_base(T &&value, error_code error) noexcept +simdjson_inline simdjson_result_base::simdjson_result_base(T &&value, error_code error) noexcept : std::pair(std::forward(value), error) {} template -simdjson_really_inline simdjson_result_base::simdjson_result_base(error_code error) noexcept +simdjson_inline simdjson_result_base::simdjson_result_base(error_code error) noexcept : simdjson_result_base(T{}, error) {} template -simdjson_really_inline simdjson_result_base::simdjson_result_base(T &&value) noexcept +simdjson_inline simdjson_result_base::simdjson_result_base(T &&value) noexcept : simdjson_result_base(std::forward(value), SUCCESS) {} template -simdjson_really_inline simdjson_result_base::simdjson_result_base() noexcept +simdjson_inline simdjson_result_base::simdjson_result_base() noexcept : simdjson_result_base(T{}, UNINITIALIZED) {} } // namespace internal @@ -3647,65 +3693,65 @@ simdjson_really_inline simdjson_result_base::simdjson_result_base() noexcept /// template -simdjson_really_inline void simdjson_result::tie(T &value, error_code &error) && noexcept { +simdjson_inline void simdjson_result::tie(T &value, error_code &error) && noexcept { std::forward>(*this).tie(value, error); } template -simdjson_warn_unused simdjson_really_inline error_code simdjson_result::get(T &value) && noexcept { +simdjson_warn_unused simdjson_inline error_code simdjson_result::get(T &value) && noexcept { return std::forward>(*this).get(value); } template -simdjson_really_inline error_code simdjson_result::error() const noexcept { +simdjson_inline error_code simdjson_result::error() const noexcept { return internal::simdjson_result_base::error(); } #if SIMDJSON_EXCEPTIONS template -simdjson_really_inline T& simdjson_result::value() & noexcept(false) { +simdjson_inline T& simdjson_result::value() & noexcept(false) { return internal::simdjson_result_base::value(); } template -simdjson_really_inline T&& simdjson_result::value() && noexcept(false) { +simdjson_inline T&& simdjson_result::value() && noexcept(false) { return std::forward>(*this).value(); } template -simdjson_really_inline T&& simdjson_result::take_value() && noexcept(false) { +simdjson_inline T&& simdjson_result::take_value() && noexcept(false) { return std::forward>(*this).take_value(); } template -simdjson_really_inline simdjson_result::operator T&&() && noexcept(false) { +simdjson_inline simdjson_result::operator T&&() && noexcept(false) { return std::forward>(*this).take_value(); } #endif // SIMDJSON_EXCEPTIONS template -simdjson_really_inline const T& simdjson_result::value_unsafe() const& noexcept { +simdjson_inline const T& simdjson_result::value_unsafe() const& noexcept { return internal::simdjson_result_base::value_unsafe(); } template -simdjson_really_inline T&& simdjson_result::value_unsafe() && noexcept { +simdjson_inline T&& simdjson_result::value_unsafe() && noexcept { return std::forward>(*this).value_unsafe(); } template -simdjson_really_inline simdjson_result::simdjson_result(T &&value, error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(T &&value, error_code error) noexcept : internal::simdjson_result_base(std::forward(value), error) {} template -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} template -simdjson_really_inline simdjson_result::simdjson_result(T &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(T &&value) noexcept : internal::simdjson_result_base(std::forward(value)) {} template -simdjson_really_inline simdjson_result::simdjson_result() noexcept +simdjson_inline simdjson_result::simdjson_result() noexcept : internal::simdjson_result_base() {} } // namespace simdjson @@ -3758,7 +3804,7 @@ inline char *allocate_padded_buffer(size_t length) noexcept { } // namespace internal -inline padded_string::padded_string() noexcept {} +inline padded_string::padded_string() noexcept = default; inline padded_string::padded_string(size_t length) noexcept : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { } @@ -3840,7 +3886,13 @@ inline simdjson_result padded_string::load(std::string_view filen } // Get the file size - if(std::fseek(fp, 0, SEEK_END) < 0) { + int ret; +#if defined(SIMDJSON_VISUAL_STUDIO) && !SIMDJSON_IS_32BITS + ret = _fseeki64(fp, 0, SEEK_END); +#else + ret = std::fseek(fp, 0, SEEK_END); +#endif // _WIN64 + if(ret < 0) { std::fclose(fp); return IO_ERROR; } @@ -3984,25 +4036,26 @@ constexpr const uint32_t JSON_COUNT_MASK = 0xFFFFFF; */ class tape_ref { public: - simdjson_really_inline tape_ref() noexcept; - simdjson_really_inline tape_ref(const dom::document *doc, size_t json_index) noexcept; + simdjson_inline tape_ref() noexcept; + simdjson_inline tape_ref(const dom::document *doc, size_t json_index) noexcept; inline size_t after_element() const noexcept; - simdjson_really_inline tape_type tape_ref_type() const noexcept; - simdjson_really_inline uint64_t tape_value() const noexcept; - simdjson_really_inline bool is_double() const noexcept; - simdjson_really_inline bool is_int64() const noexcept; - simdjson_really_inline bool is_uint64() const noexcept; - simdjson_really_inline bool is_false() const noexcept; - simdjson_really_inline bool is_true() const noexcept; - simdjson_really_inline bool is_null_on_tape() const noexcept;// different name to avoid clash with is_null. - simdjson_really_inline uint32_t matching_brace_index() const noexcept; - simdjson_really_inline uint32_t scope_count() const noexcept; + simdjson_inline tape_type tape_ref_type() const noexcept; + simdjson_inline uint64_t tape_value() const noexcept; + simdjson_inline bool is_double() const noexcept; + simdjson_inline bool is_int64() const noexcept; + simdjson_inline bool is_uint64() const noexcept; + simdjson_inline bool is_false() const noexcept; + simdjson_inline bool is_true() const noexcept; + simdjson_inline bool is_null_on_tape() const noexcept;// different name to avoid clash with is_null. + simdjson_inline uint32_t matching_brace_index() const noexcept; + simdjson_inline uint32_t scope_count() const noexcept; template - simdjson_really_inline T next_tape_value() const noexcept; - simdjson_really_inline uint32_t get_string_length() const noexcept; - simdjson_really_inline const char * get_c_str() const noexcept; + simdjson_inline T next_tape_value() const noexcept; + simdjson_inline uint32_t get_string_length() const noexcept; + simdjson_inline const char * get_c_str() const noexcept; inline std::string_view get_string_view() const noexcept; - simdjson_really_inline bool is_document_root() const noexcept; + simdjson_inline bool is_document_root() const noexcept; + simdjson_inline bool usable() const noexcept; /** The document this element references. */ const dom::document *doc; @@ -4034,7 +4087,7 @@ class element; class array { public: /** Create a new, invalid array */ - simdjson_really_inline array() noexcept; + simdjson_inline array() noexcept; class iterator { public: @@ -4074,7 +4127,7 @@ class array { iterator(const iterator&) noexcept = default; iterator& operator=(const iterator&) noexcept = default; private: - simdjson_really_inline iterator(const internal::tape_ref &tape) noexcept; + simdjson_inline iterator(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; friend class array; }; @@ -4145,7 +4198,7 @@ class array { inline simdjson_result at(size_t index) const noexcept; private: - simdjson_really_inline array(const internal::tape_ref &tape) noexcept; + simdjson_inline array(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; friend class element; friend struct simdjson_result; @@ -4160,9 +4213,9 @@ class array { template<> struct simdjson_result : public internal::simdjson_result_base { public: - simdjson_really_inline simdjson_result() noexcept; ///< @private - simdjson_really_inline simdjson_result(dom::array value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept; ///< @private + simdjson_inline simdjson_result(dom::array value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private inline simdjson_result at_pointer(std::string_view json_pointer) const noexcept; inline simdjson_result at(size_t index) const noexcept; @@ -4357,20 +4410,20 @@ class parser { * to allocate an initial capacity, call allocate() after constructing the parser. * Defaults to SIMDJSON_MAXSIZE_BYTES (the largest single document simdjson can process). */ - simdjson_really_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; + simdjson_inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; /** * Take another parser's buffers and state. * * @param other The parser to take. Its capacity is zeroed. */ - simdjson_really_inline parser(parser &&other) noexcept; + simdjson_inline parser(parser &&other) noexcept; parser(const parser &) = delete; ///< @private Disallow copying /** * Take another parser's buffers and state. * * @param other The parser to take. Its capacity is zeroed. */ - simdjson_really_inline parser &operator=(parser &&other) noexcept; + simdjson_inline parser &operator=(parser &&other) noexcept; parser &operator=(const parser &) = delete; ///< @private Disallow copying /** Deallocate the JSON parser. */ @@ -4480,17 +4533,20 @@ class parser { inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept; inline simdjson_result parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete; /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_really_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; - simdjson_really_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete; + simdjson_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; + simdjson_inline simdjson_result parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete; + /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ + simdjson_inline simdjson_result parse(const std::string &s) & noexcept; + simdjson_inline simdjson_result parse(const std::string &s) && =delete; /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_really_inline simdjson_result parse(const std::string &s) & noexcept; - simdjson_really_inline simdjson_result parse(const std::string &s) && =delete; + simdjson_inline simdjson_result parse(const padded_string &s) & noexcept; + simdjson_inline simdjson_result parse(const padded_string &s) && =delete; /** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_really_inline simdjson_result parse(const padded_string &s) & noexcept; - simdjson_really_inline simdjson_result parse(const padded_string &s) && =delete; + simdjson_inline simdjson_result parse(const padded_string_view &v) & noexcept; + simdjson_inline simdjson_result parse(const padded_string_view &v) && =delete; /** @private We do not want to allow implicit conversion from C string to std::string. */ - simdjson_really_inline simdjson_result parse(const char *buf) noexcept = delete; + simdjson_inline simdjson_result parse(const char *buf) noexcept = delete; /** * Parse a JSON document into a provide document instance and return a temporary reference to it. @@ -4539,17 +4595,17 @@ class parser { inline simdjson_result parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept; inline simdjson_result parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete; /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_really_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; - simdjson_really_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) && =delete; + simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) & noexcept; + simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) && =delete; /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_really_inline simdjson_result parse_into_document(document& doc, const std::string &s) & noexcept; - simdjson_really_inline simdjson_result parse_into_document(document& doc, const std::string &s) && =delete; + simdjson_inline simdjson_result parse_into_document(document& doc, const std::string &s) & noexcept; + simdjson_inline simdjson_result parse_into_document(document& doc, const std::string &s) && =delete; /** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */ - simdjson_really_inline simdjson_result parse_into_document(document& doc, const padded_string &s) & noexcept; - simdjson_really_inline simdjson_result parse_into_document(document& doc, const padded_string &s) && =delete; + simdjson_inline simdjson_result parse_into_document(document& doc, const padded_string &s) & noexcept; + simdjson_inline simdjson_result parse_into_document(document& doc, const padded_string &s) && =delete; /** @private We do not want to allow implicit conversion from C string to std::string. */ - simdjson_really_inline simdjson_result parse_into_document(document& doc, const char *buf) noexcept = delete; + simdjson_inline simdjson_result parse_into_document(document& doc, const char *buf) noexcept = delete; /** * Load a file containing many JSON documents. @@ -4761,7 +4817,7 @@ class parser { * * @return Current capacity, in bytes. */ - simdjson_really_inline size_t capacity() const noexcept; + simdjson_inline size_t capacity() const noexcept; /** * The largest document this parser can automatically support. @@ -4770,14 +4826,14 @@ class parser { * * @return Maximum capacity, in bytes. */ - simdjson_really_inline size_t max_capacity() const noexcept; + simdjson_inline size_t max_capacity() const noexcept; /** * The maximum level of nested object and arrays supported by this parser. * * @return Maximum depth, in bytes. */ - simdjson_really_inline size_t max_depth() const noexcept; + simdjson_inline size_t max_depth() const noexcept; /** * Set max_capacity. This is the largest document this parser can automatically support. @@ -4793,7 +4849,7 @@ class parser { * * @param max_capacity The new maximum capacity, in bytes. */ - simdjson_really_inline void set_max_capacity(size_t max_capacity) noexcept; + simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; #ifdef SIMDJSON_THREADS_ENABLED /** @@ -4975,13 +5031,13 @@ class document_stream { * error = parser.parse_many(json).get(docs); * ``` */ - simdjson_really_inline document_stream() noexcept; + simdjson_inline document_stream() noexcept; /** Move one document_stream to another. */ - simdjson_really_inline document_stream(document_stream &&other) noexcept = default; + simdjson_inline document_stream(document_stream &&other) noexcept = default; /** Move one document_stream to another. */ - simdjson_really_inline document_stream &operator=(document_stream &&other) noexcept = default; + simdjson_inline document_stream &operator=(document_stream &&other) noexcept = default; - simdjson_really_inline ~document_stream() noexcept; + simdjson_inline ~document_stream() noexcept; /** * Returns the input size in bytes. */ @@ -5020,11 +5076,11 @@ class document_stream { /** * Default constructor. */ - simdjson_really_inline iterator() noexcept; + simdjson_inline iterator() noexcept; /** * Get the current document (or error). */ - simdjson_really_inline reference operator*() noexcept; + simdjson_inline reference operator*() noexcept; /** * Advance to the next document (prefix). */ @@ -5033,7 +5089,7 @@ class document_stream { * Check if we're at the end yet. * @param other the end iterator to compare to. */ - simdjson_really_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator!=(const iterator &other) const noexcept; /** * @private * @@ -5049,7 +5105,7 @@ class document_stream { * may change in future versions of simdjson: we find the API somewhat * awkward and we would like to offer something friendlier. */ - simdjson_really_inline size_t current_index() const noexcept; + simdjson_inline size_t current_index() const noexcept; /** * @private * @@ -5069,10 +5125,10 @@ class document_stream { * may change in future versions of simdjson: we find the API somewhat * awkward and we would like to offer something friendlier. */ - simdjson_really_inline std::string_view source() const noexcept; + simdjson_inline std::string_view source() const noexcept; private: - simdjson_really_inline iterator(document_stream *s, bool finished) noexcept; + simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ document_stream* stream; /** Whether we're finished or not. */ @@ -5083,11 +5139,11 @@ class document_stream { /** * Start iterating the documents in the stream. */ - simdjson_really_inline iterator begin() noexcept; + simdjson_inline iterator begin() noexcept; /** * The end of the stream, for iterator comparison purposes. */ - simdjson_really_inline iterator end() noexcept; + simdjson_inline iterator end() noexcept; private: @@ -5103,7 +5159,7 @@ class document_stream { * @param len is the length of the raw byte buffer in bytes * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) */ - simdjson_really_inline document_stream( + simdjson_inline document_stream( dom::parser &parser, const uint8_t *buf, size_t len, @@ -5197,19 +5253,19 @@ class document_stream { template<> struct simdjson_result : public internal::simdjson_result_base { public: - simdjson_really_inline simdjson_result() noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result(dom::document_stream &&value) noexcept; ///< @private #if SIMDJSON_EXCEPTIONS - simdjson_really_inline dom::document_stream::iterator begin() noexcept(false); - simdjson_really_inline dom::document_stream::iterator end() noexcept(false); + simdjson_inline dom::document_stream::iterator begin() noexcept(false); + simdjson_inline dom::document_stream::iterator end() noexcept(false); #else // SIMDJSON_EXCEPTIONS #ifndef SIMDJSON_DISABLE_DEPRECATED_API [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] - simdjson_really_inline dom::document_stream::iterator begin() noexcept; + simdjson_inline dom::document_stream::iterator begin() noexcept; [[deprecated("parse_many() and load_many() may return errors. Use document_stream stream; error = parser.parse_many().get(doc); instead.")]] - simdjson_really_inline dom::document_stream::iterator end() noexcept; + simdjson_inline dom::document_stream::iterator end() noexcept; #endif // SIMDJSON_DISABLE_DEPRECATED_API #endif // SIMDJSON_EXCEPTIONS }; // struct simdjson_result @@ -5258,10 +5314,10 @@ enum class element_type { class element { public: /** Create a new, invalid element. */ - simdjson_really_inline element() noexcept; + simdjson_inline element() noexcept; /** The type of this element. */ - simdjson_really_inline element_type type() const noexcept; + simdjson_inline element_type type() const noexcept; /** * Cast this element to an array. @@ -5413,7 +5469,7 @@ class element { * @tparam T bool, double, uint64_t, int64_t, std::string_view, const char *, dom::array, dom::object */ template - simdjson_really_inline bool is() const noexcept; + simdjson_inline bool is() const noexcept; /** * Get the value as the provided type (T). @@ -5458,7 +5514,7 @@ class element { * @returns The error that occurred, or SUCCESS if there was no error. */ template - simdjson_warn_unused simdjson_really_inline error_code get(T &value) const noexcept; + simdjson_warn_unused simdjson_inline error_code get(T &value) const noexcept; /** * Get the value as the provided type (T), setting error if it's not the given type. @@ -5682,7 +5738,7 @@ class element { inline bool dump_raw_tape(std::ostream &out) const noexcept; private: - simdjson_really_inline element(const internal::tape_ref &tape) noexcept; + simdjson_inline element(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; friend class document; friend class object; @@ -5699,59 +5755,59 @@ class element { template<> struct simdjson_result : public internal::simdjson_result_base { public: - simdjson_really_inline simdjson_result() noexcept; ///< @private - simdjson_really_inline simdjson_result(dom::element &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept; ///< @private + simdjson_inline simdjson_result(dom::element &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result type() const noexcept; + simdjson_inline simdjson_result type() const noexcept; template - simdjson_really_inline bool is() const noexcept; + simdjson_inline bool is() const noexcept; template - simdjson_really_inline simdjson_result get() const noexcept; + simdjson_inline simdjson_result get() const noexcept; template - simdjson_warn_unused simdjson_really_inline error_code get(T &value) const noexcept; - - simdjson_really_inline simdjson_result get_array() const noexcept; - simdjson_really_inline simdjson_result get_object() const noexcept; - simdjson_really_inline simdjson_result get_c_str() const noexcept; - simdjson_really_inline simdjson_result get_string_length() const noexcept; - simdjson_really_inline simdjson_result get_string() const noexcept; - simdjson_really_inline simdjson_result get_int64() const noexcept; - simdjson_really_inline simdjson_result get_uint64() const noexcept; - simdjson_really_inline simdjson_result get_double() const noexcept; - simdjson_really_inline simdjson_result get_bool() const noexcept; - - simdjson_really_inline bool is_array() const noexcept; - simdjson_really_inline bool is_object() const noexcept; - simdjson_really_inline bool is_string() const noexcept; - simdjson_really_inline bool is_int64() const noexcept; - simdjson_really_inline bool is_uint64() const noexcept; - simdjson_really_inline bool is_double() const noexcept; - simdjson_really_inline bool is_number() const noexcept; - simdjson_really_inline bool is_bool() const noexcept; - simdjson_really_inline bool is_null() const noexcept; - - simdjson_really_inline simdjson_result operator[](std::string_view key) const noexcept; - simdjson_really_inline simdjson_result operator[](const char *key) const noexcept; - simdjson_really_inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; + simdjson_warn_unused simdjson_inline error_code get(T &value) const noexcept; + + simdjson_inline simdjson_result get_array() const noexcept; + simdjson_inline simdjson_result get_object() const noexcept; + simdjson_inline simdjson_result get_c_str() const noexcept; + simdjson_inline simdjson_result get_string_length() const noexcept; + simdjson_inline simdjson_result get_string() const noexcept; + simdjson_inline simdjson_result get_int64() const noexcept; + simdjson_inline simdjson_result get_uint64() const noexcept; + simdjson_inline simdjson_result get_double() const noexcept; + simdjson_inline simdjson_result get_bool() const noexcept; + + simdjson_inline bool is_array() const noexcept; + simdjson_inline bool is_object() const noexcept; + simdjson_inline bool is_string() const noexcept; + simdjson_inline bool is_int64() const noexcept; + simdjson_inline bool is_uint64() const noexcept; + simdjson_inline bool is_double() const noexcept; + simdjson_inline bool is_number() const noexcept; + simdjson_inline bool is_bool() const noexcept; + simdjson_inline bool is_null() const noexcept; + + simdjson_inline simdjson_result operator[](std::string_view key) const noexcept; + simdjson_inline simdjson_result operator[](const char *key) const noexcept; + simdjson_inline simdjson_result at_pointer(const std::string_view json_pointer) const noexcept; [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] - simdjson_really_inline simdjson_result at(const std::string_view json_pointer) const noexcept; - simdjson_really_inline simdjson_result at(size_t index) const noexcept; - simdjson_really_inline simdjson_result at_key(std::string_view key) const noexcept; - simdjson_really_inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; + simdjson_inline simdjson_result at(const std::string_view json_pointer) const noexcept; + simdjson_inline simdjson_result at(size_t index) const noexcept; + simdjson_inline simdjson_result at_key(std::string_view key) const noexcept; + simdjson_inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; #if SIMDJSON_EXCEPTIONS - simdjson_really_inline operator bool() const noexcept(false); - simdjson_really_inline explicit operator const char*() const noexcept(false); - simdjson_really_inline operator std::string_view() const noexcept(false); - simdjson_really_inline operator uint64_t() const noexcept(false); - simdjson_really_inline operator int64_t() const noexcept(false); - simdjson_really_inline operator double() const noexcept(false); - simdjson_really_inline operator dom::array() const noexcept(false); - simdjson_really_inline operator dom::object() const noexcept(false); - - simdjson_really_inline dom::array::iterator begin() const noexcept(false); - simdjson_really_inline dom::array::iterator end() const noexcept(false); + simdjson_inline operator bool() const noexcept(false); + simdjson_inline explicit operator const char*() const noexcept(false); + simdjson_inline operator std::string_view() const noexcept(false); + simdjson_inline operator uint64_t() const noexcept(false); + simdjson_inline operator int64_t() const noexcept(false); + simdjson_inline operator double() const noexcept(false); + simdjson_inline operator dom::array() const noexcept(false); + simdjson_inline operator dom::object() const noexcept(false); + + simdjson_inline dom::array::iterator begin() const noexcept(false); + simdjson_inline dom::array::iterator end() const noexcept(false); #endif // SIMDJSON_EXCEPTIONS }; @@ -5782,7 +5838,7 @@ class key_value_pair; class object { public: /** Create a new, invalid object */ - simdjson_really_inline object() noexcept; + simdjson_inline object() noexcept; class iterator { public: @@ -5852,7 +5908,7 @@ class object { iterator(const iterator&) noexcept = default; iterator& operator=(const iterator&) noexcept = default; private: - simdjson_really_inline iterator(const internal::tape_ref &tape) noexcept; + simdjson_inline iterator(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; @@ -5966,7 +6022,7 @@ class object { inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; private: - simdjson_really_inline object(const internal::tape_ref &tape) noexcept; + simdjson_inline object(const internal::tape_ref &tape) noexcept; internal::tape_ref tape; @@ -5987,7 +6043,7 @@ class key_value_pair { element value; private: - simdjson_really_inline key_value_pair(std::string_view _key, element _value) noexcept; + simdjson_inline key_value_pair(std::string_view _key, element _value) noexcept; friend class object; }; @@ -5997,9 +6053,9 @@ class key_value_pair { template<> struct simdjson_result : public internal::simdjson_result_base { public: - simdjson_really_inline simdjson_result() noexcept; ///< @private - simdjson_really_inline simdjson_result(dom::object value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept; ///< @private + simdjson_inline simdjson_result(dom::object value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private inline simdjson_result operator[](std::string_view key) const noexcept; inline simdjson_result operator[](const char *key) const noexcept; @@ -6069,7 +6125,7 @@ class string_builder { /** Append an object to the builder (to be printed) **/ inline void append(simdjson::dom::object value); /** Reset the builder (so that it would print the empty string) **/ - simdjson_really_inline void clear(); + simdjson_inline void clear(); /** * Get access to the string. The string_view is owned by the builder * and it is invalid to use it after the string_builder has been @@ -6077,9 +6133,9 @@ class string_builder { * However you can make a copy of the string_view on memory that you * own. */ - simdjson_really_inline std::string_view str() const; + simdjson_inline std::string_view str() const; /** Append a key_value_pair to the builder (to be printed) **/ - simdjson_really_inline void append(simdjson::dom::key_value_pair value); + simdjson_inline void append(simdjson::dom::key_value_pair value); private: formatter format{}; }; @@ -6093,43 +6149,43 @@ class mini_formatter { public: mini_formatter() = default; /** Add a comma **/ - simdjson_really_inline void comma(); + simdjson_inline void comma(); /** Start an array, prints [ **/ - simdjson_really_inline void start_array(); + simdjson_inline void start_array(); /** End an array, prints ] **/ - simdjson_really_inline void end_array(); + simdjson_inline void end_array(); /** Start an array, prints { **/ - simdjson_really_inline void start_object(); + simdjson_inline void start_object(); /** Start an array, prints } **/ - simdjson_really_inline void end_object(); + simdjson_inline void end_object(); /** Prints a true **/ - simdjson_really_inline void true_atom(); + simdjson_inline void true_atom(); /** Prints a false **/ - simdjson_really_inline void false_atom(); + simdjson_inline void false_atom(); /** Prints a null **/ - simdjson_really_inline void null_atom(); + simdjson_inline void null_atom(); /** Prints a number **/ - simdjson_really_inline void number(int64_t x); + simdjson_inline void number(int64_t x); /** Prints a number **/ - simdjson_really_inline void number(uint64_t x); + simdjson_inline void number(uint64_t x); /** Prints a number **/ - simdjson_really_inline void number(double x); + simdjson_inline void number(double x); /** Prints a key (string + colon) **/ - simdjson_really_inline void key(std::string_view unescaped); + simdjson_inline void key(std::string_view unescaped); /** Prints a string. The string is escaped as needed. **/ - simdjson_really_inline void string(std::string_view unescaped); + simdjson_inline void string(std::string_view unescaped); /** Clears out the content. **/ - simdjson_really_inline void clear(); + simdjson_inline void clear(); /** * Get access to the buffer, it is owned by the instance, but * the user can make a copy. **/ - simdjson_really_inline std::string_view str() const; + simdjson_inline std::string_view str() const; private: // implementation details (subject to change) /** Prints one character **/ - simdjson_really_inline void one_char(char c); + simdjson_inline void one_char(char c); /** Backing buffer **/ std::vector buffer{}; // not ideal! }; @@ -6758,11 +6814,11 @@ namespace simdjson { // // simdjson_result inline implementation // -simdjson_really_inline simdjson_result::simdjson_result() noexcept +simdjson_inline simdjson_result::simdjson_result() noexcept : internal::simdjson_result_base() {} -simdjson_really_inline simdjson_result::simdjson_result(dom::array value) noexcept +simdjson_inline simdjson_result::simdjson_result(dom::array value) noexcept : internal::simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} #if SIMDJSON_EXCEPTIONS @@ -6796,21 +6852,26 @@ namespace dom { // // array inline implementation // -simdjson_really_inline array::array() noexcept : tape{} {} -simdjson_really_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {} +simdjson_inline array::array() noexcept : tape{} {} +simdjson_inline array::array(const internal::tape_ref &_tape) noexcept : tape{_tape} {} inline array::iterator array::begin() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return internal::tape_ref(tape.doc, tape.json_index + 1); } inline array::iterator array::end() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return internal::tape_ref(tape.doc, tape.after_element() - 1); } inline size_t array::size() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return tape.scope_count(); } inline size_t array::number_of_slots() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return tape.matching_brace_index() - tape.json_index; } inline simdjson_result array::at_pointer(std::string_view json_pointer) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 if(json_pointer.empty()) { // an empty string means that we return the current node return element(this->tape); // copy the current node } else if(json_pointer[0] != '/') { // otherwise there is an error @@ -6851,6 +6912,7 @@ inline simdjson_result array::at_pointer(std::string_view json_pointer) } inline simdjson_result array::at(size_t index) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 size_t i=0; for (auto element : *this) { if (i == index) { return element; } @@ -6862,7 +6924,7 @@ inline simdjson_result array::at(size_t index) const noexcept { // // array::iterator inline implementation // -simdjson_really_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +simdjson_inline array::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline element array::iterator::operator*() const noexcept { return element(tape); } @@ -6911,11 +6973,11 @@ namespace simdjson { // // simdjson_result inline implementation // -simdjson_really_inline simdjson_result::simdjson_result() noexcept +simdjson_inline simdjson_result::simdjson_result() noexcept : internal::simdjson_result_base() {} -simdjson_really_inline simdjson_result::simdjson_result(dom::element &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(dom::element &&value) noexcept : internal::simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} inline simdjson_result simdjson_result::type() const noexcept { if (error()) { return error(); } @@ -6923,101 +6985,101 @@ inline simdjson_result simdjson_result::type() } template -simdjson_really_inline bool simdjson_result::is() const noexcept { +simdjson_inline bool simdjson_result::is() const noexcept { return !error() && first.is(); } template -simdjson_really_inline simdjson_result simdjson_result::get() const noexcept { +simdjson_inline simdjson_result simdjson_result::get() const noexcept { if (error()) { return error(); } return first.get(); } template -simdjson_warn_unused simdjson_really_inline error_code simdjson_result::get(T &value) const noexcept { +simdjson_warn_unused simdjson_inline error_code simdjson_result::get(T &value) const noexcept { if (error()) { return error(); } return first.get(value); } -simdjson_really_inline simdjson_result simdjson_result::get_array() const noexcept { +simdjson_inline simdjson_result simdjson_result::get_array() const noexcept { if (error()) { return error(); } return first.get_array(); } -simdjson_really_inline simdjson_result simdjson_result::get_object() const noexcept { +simdjson_inline simdjson_result simdjson_result::get_object() const noexcept { if (error()) { return error(); } return first.get_object(); } -simdjson_really_inline simdjson_result simdjson_result::get_c_str() const noexcept { +simdjson_inline simdjson_result simdjson_result::get_c_str() const noexcept { if (error()) { return error(); } return first.get_c_str(); } -simdjson_really_inline simdjson_result simdjson_result::get_string_length() const noexcept { +simdjson_inline simdjson_result simdjson_result::get_string_length() const noexcept { if (error()) { return error(); } return first.get_string_length(); } -simdjson_really_inline simdjson_result simdjson_result::get_string() const noexcept { +simdjson_inline simdjson_result simdjson_result::get_string() const noexcept { if (error()) { return error(); } return first.get_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_int64() const noexcept { +simdjson_inline simdjson_result simdjson_result::get_int64() const noexcept { if (error()) { return error(); } return first.get_int64(); } -simdjson_really_inline simdjson_result simdjson_result::get_uint64() const noexcept { +simdjson_inline simdjson_result simdjson_result::get_uint64() const noexcept { if (error()) { return error(); } return first.get_uint64(); } -simdjson_really_inline simdjson_result simdjson_result::get_double() const noexcept { +simdjson_inline simdjson_result simdjson_result::get_double() const noexcept { if (error()) { return error(); } return first.get_double(); } -simdjson_really_inline simdjson_result simdjson_result::get_bool() const noexcept { +simdjson_inline simdjson_result simdjson_result::get_bool() const noexcept { if (error()) { return error(); } return first.get_bool(); } -simdjson_really_inline bool simdjson_result::is_array() const noexcept { +simdjson_inline bool simdjson_result::is_array() const noexcept { return !error() && first.is_array(); } -simdjson_really_inline bool simdjson_result::is_object() const noexcept { +simdjson_inline bool simdjson_result::is_object() const noexcept { return !error() && first.is_object(); } -simdjson_really_inline bool simdjson_result::is_string() const noexcept { +simdjson_inline bool simdjson_result::is_string() const noexcept { return !error() && first.is_string(); } -simdjson_really_inline bool simdjson_result::is_int64() const noexcept { +simdjson_inline bool simdjson_result::is_int64() const noexcept { return !error() && first.is_int64(); } -simdjson_really_inline bool simdjson_result::is_uint64() const noexcept { +simdjson_inline bool simdjson_result::is_uint64() const noexcept { return !error() && first.is_uint64(); } -simdjson_really_inline bool simdjson_result::is_double() const noexcept { +simdjson_inline bool simdjson_result::is_double() const noexcept { return !error() && first.is_double(); } -simdjson_really_inline bool simdjson_result::is_number() const noexcept { +simdjson_inline bool simdjson_result::is_number() const noexcept { return !error() && first.is_number(); } -simdjson_really_inline bool simdjson_result::is_bool() const noexcept { +simdjson_inline bool simdjson_result::is_bool() const noexcept { return !error() && first.is_bool(); } -simdjson_really_inline bool simdjson_result::is_null() const noexcept { +simdjson_inline bool simdjson_result::is_null() const noexcept { return !error() && first.is_null(); } -simdjson_really_inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { if (error()) { return error(); } return first[key]; } -simdjson_really_inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](const char *key) const noexcept { if (error()) { return error(); } return first[key]; } -simdjson_really_inline simdjson_result simdjson_result::at_pointer(const std::string_view json_pointer) const noexcept { +simdjson_inline simdjson_result simdjson_result::at_pointer(const std::string_view json_pointer) const noexcept { if (error()) { return error(); } return first.at_pointer(json_pointer); } #ifndef SIMDJSON_DISABLE_DEPRECATED_API [[deprecated("For standard compliance, use at_pointer instead, and prefix your pointers with a slash '/', see RFC6901 ")]] -simdjson_really_inline simdjson_result simdjson_result::at(const std::string_view json_pointer) const noexcept { +simdjson_inline simdjson_result simdjson_result::at(const std::string_view json_pointer) const noexcept { SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_DEPRECATED_WARNING if (error()) { return error(); } @@ -7025,51 +7087,51 @@ SIMDJSON_DISABLE_DEPRECATED_WARNING SIMDJSON_POP_DISABLE_WARNINGS } #endif // SIMDJSON_DISABLE_DEPRECATED_API -simdjson_really_inline simdjson_result simdjson_result::at(size_t index) const noexcept { +simdjson_inline simdjson_result simdjson_result::at(size_t index) const noexcept { if (error()) { return error(); } return first.at(index); } -simdjson_really_inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { +simdjson_inline simdjson_result simdjson_result::at_key(std::string_view key) const noexcept { if (error()) { return error(); } return first.at_key(key); } -simdjson_really_inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { +simdjson_inline simdjson_result simdjson_result::at_key_case_insensitive(std::string_view key) const noexcept { if (error()) { return error(); } return first.at_key_case_insensitive(key); } #if SIMDJSON_EXCEPTIONS -simdjson_really_inline simdjson_result::operator bool() const noexcept(false) { +simdjson_inline simdjson_result::operator bool() const noexcept(false) { return get(); } -simdjson_really_inline simdjson_result::operator const char *() const noexcept(false) { +simdjson_inline simdjson_result::operator const char *() const noexcept(false) { return get(); } -simdjson_really_inline simdjson_result::operator std::string_view() const noexcept(false) { +simdjson_inline simdjson_result::operator std::string_view() const noexcept(false) { return get(); } -simdjson_really_inline simdjson_result::operator uint64_t() const noexcept(false) { +simdjson_inline simdjson_result::operator uint64_t() const noexcept(false) { return get(); } -simdjson_really_inline simdjson_result::operator int64_t() const noexcept(false) { +simdjson_inline simdjson_result::operator int64_t() const noexcept(false) { return get(); } -simdjson_really_inline simdjson_result::operator double() const noexcept(false) { +simdjson_inline simdjson_result::operator double() const noexcept(false) { return get(); } -simdjson_really_inline simdjson_result::operator dom::array() const noexcept(false) { +simdjson_inline simdjson_result::operator dom::array() const noexcept(false) { return get(); } -simdjson_really_inline simdjson_result::operator dom::object() const noexcept(false) { +simdjson_inline simdjson_result::operator dom::object() const noexcept(false) { return get(); } -simdjson_really_inline dom::array::iterator simdjson_result::begin() const noexcept(false) { +simdjson_inline dom::array::iterator simdjson_result::begin() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.begin(); } -simdjson_really_inline dom::array::iterator simdjson_result::end() const noexcept(false) { +simdjson_inline dom::array::iterator simdjson_result::end() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.end(); } @@ -7081,15 +7143,17 @@ namespace dom { // // element inline implementation // -simdjson_really_inline element::element() noexcept : tape{} {} -simdjson_really_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +simdjson_inline element::element() noexcept : tape{} {} +simdjson_inline element::element(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline element_type element::type() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 auto tape_type = tape.tape_ref_type(); return tape_type == internal::tape_type::FALSE_VALUE ? element_type::BOOL : static_cast(tape_type); } inline simdjson_result element::get_bool() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 if(tape.is_true()) { return true; } else if(tape.is_false()) { @@ -7098,6 +7162,7 @@ inline simdjson_result element::get_bool() const noexcept { return INCORRECT_TYPE; } inline simdjson_result element::get_c_str() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 switch (tape.tape_ref_type()) { case internal::tape_type::STRING: { return tape.get_c_str(); @@ -7107,6 +7172,7 @@ inline simdjson_result element::get_c_str() const noexcept { } } inline simdjson_result element::get_string_length() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 switch (tape.tape_ref_type()) { case internal::tape_type::STRING: { return tape.get_string_length(); @@ -7116,6 +7182,7 @@ inline simdjson_result element::get_string_length() const noexcept { } } inline simdjson_result element::get_string() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 switch (tape.tape_ref_type()) { case internal::tape_type::STRING: return tape.get_string_view(); @@ -7124,6 +7191,7 @@ inline simdjson_result element::get_string() const noexcept { } } inline simdjson_result element::get_uint64() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 if(simdjson_unlikely(!tape.is_uint64())) { // branch rarely taken if(tape.is_int64()) { int64_t result = tape.next_tape_value(); @@ -7137,6 +7205,7 @@ inline simdjson_result element::get_uint64() const noexcept { return tape.next_tape_value(); } inline simdjson_result element::get_int64() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 if(simdjson_unlikely(!tape.is_int64())) { // branch rarely taken if(tape.is_uint64()) { uint64_t result = tape.next_tape_value(); @@ -7151,6 +7220,7 @@ inline simdjson_result element::get_int64() const noexcept { return tape.next_tape_value(); } inline simdjson_result element::get_double() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 // Performance considerations: // 1. Querying tape_ref_type() implies doing a shift, it is fast to just do a straight // comparison. @@ -7172,6 +7242,7 @@ inline simdjson_result element::get_double() const noexcept { return tape.next_tape_value(); } inline simdjson_result element::get_array() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 switch (tape.tape_ref_type()) { case internal::tape_type::START_ARRAY: return array(tape); @@ -7180,6 +7251,7 @@ inline simdjson_result element::get_array() const noexcept { } } inline simdjson_result element::get_object() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 switch (tape.tape_ref_type()) { case internal::tape_type::START_OBJECT: return object(tape); @@ -7189,12 +7261,12 @@ inline simdjson_result element::get_object() const noexcept { } template -simdjson_warn_unused simdjson_really_inline error_code element::get(T &value) const noexcept { +simdjson_warn_unused simdjson_inline error_code element::get(T &value) const noexcept { return get().get(value); } // An element-specific version prevents recursion with simdjson_result::get(value) template<> -simdjson_warn_unused simdjson_really_inline error_code element::get(element &value) const noexcept { +simdjson_warn_unused simdjson_inline error_code element::get(element &value) const noexcept { value = element(tape); return SUCCESS; } @@ -7204,7 +7276,7 @@ inline void element::tie(T &value, error_code &error) && noexcept { } template -simdjson_really_inline bool element::is() const noexcept { +simdjson_inline bool element::is() const noexcept { auto result = get(); return !result.error(); } @@ -7259,6 +7331,7 @@ inline simdjson_result element::operator[](const char *key) const noexc } inline simdjson_result element::at_pointer(std::string_view json_pointer) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 switch (tape.tape_ref_type()) { case internal::tape_type::START_OBJECT: return object(tape).at_pointer(json_pointer); @@ -7294,6 +7367,7 @@ inline simdjson_result element::at_key_case_insensitive(std::string_vie } inline bool element::dump_raw_tape(std::ostream &out) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return tape.doc->dump_raw_tape(out); } @@ -7422,7 +7496,7 @@ inline void stage1_worker::run(document_stream * ds, dom::parser * stage1, size_ } #endif -simdjson_really_inline document_stream::document_stream( +simdjson_inline document_stream::document_stream( dom::parser &_parser, const uint8_t *_buf, size_t _len, @@ -7444,7 +7518,7 @@ simdjson_really_inline document_stream::document_stream( #endif } -simdjson_really_inline document_stream::document_stream() noexcept +simdjson_inline document_stream::document_stream() noexcept : parser{nullptr}, buf{nullptr}, len{0}, @@ -7456,31 +7530,31 @@ simdjson_really_inline document_stream::document_stream() noexcept { } -simdjson_really_inline document_stream::~document_stream() noexcept { +simdjson_inline document_stream::~document_stream() noexcept { #ifdef SIMDJSON_THREADS_ENABLED worker.reset(); #endif } -simdjson_really_inline document_stream::iterator::iterator() noexcept +simdjson_inline document_stream::iterator::iterator() noexcept : stream{nullptr}, finished{true} { } -simdjson_really_inline document_stream::iterator document_stream::begin() noexcept { +simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. return iterator(this, error == EMPTY); } -simdjson_really_inline document_stream::iterator document_stream::end() noexcept { +simdjson_inline document_stream::iterator document_stream::end() noexcept { return iterator(this, true); } -simdjson_really_inline document_stream::iterator::iterator(document_stream* _stream, bool is_end) noexcept +simdjson_inline document_stream::iterator::iterator(document_stream* _stream, bool is_end) noexcept : stream{_stream}, finished{is_end} { } -simdjson_really_inline document_stream::iterator::reference document_stream::iterator::operator*() noexcept { +simdjson_inline document_stream::iterator::reference document_stream::iterator::operator*() noexcept { // Note that in case of error, we do not yet mark // the iterator as "finished": this detection is done // in the operator++ function since it is possible @@ -7490,7 +7564,7 @@ simdjson_really_inline document_stream::iterator::reference document_stream::ite return stream->parser->doc.root(); } -simdjson_really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { +simdjson_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { // If there is an error, then we want the iterator // to be finished, no matter what. (E.g., we do not // keep generating documents with errors, or go beyond @@ -7517,7 +7591,7 @@ simdjson_really_inline document_stream::iterator& document_stream::iterator::ope return *this; } -simdjson_really_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { +simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } @@ -7548,11 +7622,11 @@ inline void document_stream::start() noexcept { next(); } -simdjson_really_inline size_t document_stream::iterator::current_index() const noexcept { +simdjson_inline size_t document_stream::iterator::current_index() const noexcept { return stream->doc_index; } -simdjson_really_inline std::string_view document_stream::iterator::source() const noexcept { +simdjson_inline std::string_view document_stream::iterator::source() const noexcept { const char* start = reinterpret_cast(stream->buf) + current_index(); bool object_or_array = ((*start == '[') || (*start == '{')); if(object_or_array) { @@ -7645,32 +7719,32 @@ inline void document_stream::start_stage1_thread() noexcept { } // namespace dom -simdjson_really_inline simdjson_result::simdjson_result() noexcept +simdjson_inline simdjson_result::simdjson_result() noexcept : simdjson_result_base() { } -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : simdjson_result_base(error) { } -simdjson_really_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(dom::document_stream &&value) noexcept : simdjson_result_base(std::forward(value)) { } #if SIMDJSON_EXCEPTIONS -simdjson_really_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { +simdjson_inline dom::document_stream::iterator simdjson_result::begin() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.begin(); } -simdjson_really_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { +simdjson_inline dom::document_stream::iterator simdjson_result::end() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.end(); } #else // SIMDJSON_EXCEPTIONS #ifndef SIMDJSON_DISABLE_DEPRECATED_API -simdjson_really_inline dom::document_stream::iterator simdjson_result::begin() noexcept { +simdjson_inline dom::document_stream::iterator simdjson_result::begin() noexcept { first.error = error(); return first.begin(); } -simdjson_really_inline dom::document_stream::iterator simdjson_result::end() noexcept { +simdjson_inline dom::document_stream::iterator simdjson_result::end() noexcept { first.error = error(); return first.end(); } @@ -7848,11 +7922,11 @@ namespace simdjson { // // simdjson_result inline implementation // -simdjson_really_inline simdjson_result::simdjson_result() noexcept +simdjson_inline simdjson_result::simdjson_result() noexcept : internal::simdjson_result_base() {} -simdjson_really_inline simdjson_result::simdjson_result(dom::object value) noexcept +simdjson_inline simdjson_result::simdjson_result(dom::object value) noexcept : internal::simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : internal::simdjson_result_base(error) {} inline simdjson_result simdjson_result::operator[](std::string_view key) const noexcept { @@ -7898,15 +7972,18 @@ namespace dom { // // object inline implementation // -simdjson_really_inline object::object() noexcept : tape{} {} -simdjson_really_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +simdjson_inline object::object() noexcept : tape{} {} +simdjson_inline object::object(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline object::iterator object::begin() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return internal::tape_ref(tape.doc, tape.json_index + 1); } inline object::iterator object::end() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return internal::tape_ref(tape.doc, tape.after_element() - 1); } inline size_t object::size() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return tape.scope_count(); } @@ -7917,6 +7994,7 @@ inline simdjson_result object::operator[](const char *key) const noexce return at_key(key); } inline simdjson_result object::at_pointer(std::string_view json_pointer) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 if(json_pointer.empty()) { // an empty string means that we return the current node return element(this->tape); // copy the current node } else if(json_pointer[0] != '/') { // otherwise there is an error @@ -7985,7 +8063,7 @@ inline simdjson_result object::at_key_case_insensitive(std::string_view // // object::iterator inline implementation // -simdjson_really_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } +simdjson_inline object::iterator::iterator(const internal::tape_ref &_tape) noexcept : tape{_tape} { } inline const key_value_pair object::iterator::operator*() const noexcept { return key_value_pair(key(), value()); } @@ -8589,12 +8667,12 @@ namespace dom { // // parser inline implementation // -simdjson_really_inline parser::parser(size_t max_capacity) noexcept +simdjson_inline parser::parser(size_t max_capacity) noexcept : _max_capacity{max_capacity}, loaded_bytes(nullptr) { } -simdjson_really_inline parser::parser(parser &&other) noexcept = default; -simdjson_really_inline parser &parser::operator=(parser &&other) noexcept = default; +simdjson_inline parser::parser(parser &&other) noexcept = default; +simdjson_inline parser &parser::operator=(parser &&other) noexcept = default; inline bool parser::is_valid() const noexcept { return valid; } inline int parser::get_error_code() const noexcept { return error; } @@ -8616,7 +8694,13 @@ inline simdjson_result parser::read_file(const std::string &path) noexce } // Get the file size - if(std::fseek(fp, 0, SEEK_END) < 0) { + int ret; +#if defined(SIMDJSON_VISUAL_STUDIO) && !SIMDJSON_IS_32BITS + ret = _fseeki64(fp, 0, SEEK_END); +#else + ret = std::fseek(fp, 0, SEEK_END); +#endif // _WIN64 + if(ret < 0) { std::fclose(fp); return IO_ERROR; } @@ -8692,13 +8776,13 @@ inline simdjson_result parser::parse_into_document(document& provided_d return provided_doc.root(); } -simdjson_really_inline simdjson_result parser::parse_into_document(document& provided_doc, const char *buf, size_t len, bool realloc_if_needed) & noexcept { +simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const char *buf, size_t len, bool realloc_if_needed) & noexcept { return parse_into_document(provided_doc, reinterpret_cast(buf), len, realloc_if_needed); } -simdjson_really_inline simdjson_result parser::parse_into_document(document& provided_doc, const std::string &s) & noexcept { +simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const std::string &s) & noexcept { return parse_into_document(provided_doc, s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); } -simdjson_really_inline simdjson_result parser::parse_into_document(document& provided_doc, const padded_string &s) & noexcept { +simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const padded_string &s) & noexcept { return parse_into_document(provided_doc, s.data(), s.length(), false); } @@ -8707,15 +8791,18 @@ inline simdjson_result parser::parse(const uint8_t *buf, size_t len, bo return parse_into_document(doc, buf, len, realloc_if_needed); } -simdjson_really_inline simdjson_result parser::parse(const char *buf, size_t len, bool realloc_if_needed) & noexcept { +simdjson_inline simdjson_result parser::parse(const char *buf, size_t len, bool realloc_if_needed) & noexcept { return parse(reinterpret_cast(buf), len, realloc_if_needed); } -simdjson_really_inline simdjson_result parser::parse(const std::string &s) & noexcept { +simdjson_inline simdjson_result parser::parse(const std::string &s) & noexcept { return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); } -simdjson_really_inline simdjson_result parser::parse(const padded_string &s) & noexcept { +simdjson_inline simdjson_result parser::parse(const padded_string &s) & noexcept { return parse(s.data(), s.length(), false); } +simdjson_inline simdjson_result parser::parse(const padded_string_view &v) & noexcept { + return parse(v.data(), v.length(), false); +} inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } @@ -8731,13 +8818,13 @@ inline simdjson_result parser::parse_many(const padded_string & return parse_many(s.data(), s.length(), batch_size); } -simdjson_really_inline size_t parser::capacity() const noexcept { +simdjson_inline size_t parser::capacity() const noexcept { return implementation ? implementation->capacity() : 0; } -simdjson_really_inline size_t parser::max_capacity() const noexcept { +simdjson_inline size_t parser::max_capacity() const noexcept { return _max_capacity; } -simdjson_really_inline size_t parser::max_depth() const noexcept { +simdjson_inline size_t parser::max_depth() const noexcept { return implementation ? implementation->max_depth() : DEFAULT_MAX_DEPTH; } @@ -8790,7 +8877,7 @@ inline error_code parser::ensure_capacity(document& target_document, size_t desi return SUCCESS; } -simdjson_really_inline void parser::set_max_capacity(size_t max_capacity) noexcept { +simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { if(max_capacity < MINIMAL_DOCUMENT_CAPACITY) { _max_capacity = max_capacity; } else { @@ -8815,39 +8902,41 @@ namespace internal { // // tape_ref inline implementation // -simdjson_really_inline tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} -simdjson_really_inline tape_ref::tape_ref(const dom::document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} +simdjson_inline tape_ref::tape_ref() noexcept : doc{nullptr}, json_index{0} {} +simdjson_inline tape_ref::tape_ref(const dom::document *_doc, size_t _json_index) noexcept : doc{_doc}, json_index{_json_index} {} -simdjson_really_inline bool tape_ref::is_document_root() const noexcept { +simdjson_inline bool tape_ref::is_document_root() const noexcept { return json_index == 1; // should we ever change the structure of the tape, this should get updated. } - +simdjson_inline bool tape_ref::usable() const noexcept { + return doc != nullptr; // when the document pointer is null, this tape_ref is uninitialized (should not be accessed). +} // Some value types have a specific on-tape word value. It can be faster // to check the type by doing a word-to-word comparison instead of extracting the // most significant 8 bits. -simdjson_really_inline bool tape_ref::is_double() const noexcept { +simdjson_inline bool tape_ref::is_double() const noexcept { constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE)<<56; return doc->tape[json_index] == tape_double; } -simdjson_really_inline bool tape_ref::is_int64() const noexcept { +simdjson_inline bool tape_ref::is_int64() const noexcept { constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64)<<56; return doc->tape[json_index] == tape_int64; } -simdjson_really_inline bool tape_ref::is_uint64() const noexcept { +simdjson_inline bool tape_ref::is_uint64() const noexcept { constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64)<<56; return doc->tape[json_index] == tape_uint64; } -simdjson_really_inline bool tape_ref::is_false() const noexcept { +simdjson_inline bool tape_ref::is_false() const noexcept { constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE)<<56; return doc->tape[json_index] == tape_false; } -simdjson_really_inline bool tape_ref::is_true() const noexcept { +simdjson_inline bool tape_ref::is_true() const noexcept { constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE)<<56; return doc->tape[json_index] == tape_true; } -simdjson_really_inline bool tape_ref::is_null_on_tape() const noexcept { +simdjson_inline bool tape_ref::is_null_on_tape() const noexcept { constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE)<<56; return doc->tape[json_index] == tape_null; } @@ -8865,21 +8954,21 @@ inline size_t tape_ref::after_element() const noexcept { return json_index + 1; } } -simdjson_really_inline tape_type tape_ref::tape_ref_type() const noexcept { +simdjson_inline tape_type tape_ref::tape_ref_type() const noexcept { return static_cast(doc->tape[json_index] >> 56); } -simdjson_really_inline uint64_t internal::tape_ref::tape_value() const noexcept { +simdjson_inline uint64_t internal::tape_ref::tape_value() const noexcept { return doc->tape[json_index] & internal::JSON_VALUE_MASK; } -simdjson_really_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept { +simdjson_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept { return uint32_t(doc->tape[json_index]); } -simdjson_really_inline uint32_t internal::tape_ref::scope_count() const noexcept { +simdjson_inline uint32_t internal::tape_ref::scope_count() const noexcept { return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK); } template -simdjson_really_inline T tape_ref::next_tape_value() const noexcept { +simdjson_inline T tape_ref::next_tape_value() const noexcept { static_assert(sizeof(T) == sizeof(uint64_t), "next_tape_value() template parameter must be 64-bit"); // Though the following is tempting... // return *reinterpret_cast(&doc->tape[json_index + 1]); @@ -8890,14 +8979,14 @@ simdjson_really_inline T tape_ref::next_tape_value() const noexcept { return x; } -simdjson_really_inline uint32_t internal::tape_ref::get_string_length() const noexcept { +simdjson_inline uint32_t internal::tape_ref::get_string_length() const noexcept { size_t string_buf_index = size_t(tape_value()); uint32_t len; std::memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); return len; } -simdjson_really_inline const char * internal::tape_ref::get_c_str() const noexcept { +simdjson_inline const char * internal::tape_ref::get_c_str() const noexcept { size_t string_buf_index = size_t(tape_value()); return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); } @@ -9023,19 +9112,19 @@ namespace internal { * Minifier/formatter code. **/ -simdjson_really_inline void mini_formatter::number(uint64_t x) { +simdjson_inline void mini_formatter::number(uint64_t x) { char number_buffer[24]; char *newp = fast_itoa(number_buffer, x); buffer.insert(buffer.end(), number_buffer, newp); } -simdjson_really_inline void mini_formatter::number(int64_t x) { +simdjson_inline void mini_formatter::number(int64_t x) { char number_buffer[24]; char *newp = fast_itoa(number_buffer, x); buffer.insert(buffer.end(), number_buffer, newp); } -simdjson_really_inline void mini_formatter::number(double x) { +simdjson_inline void mini_formatter::number(double x) { char number_buffer[24]; // Currently, passing the nullptr to the second argument is // safe because our implementation does not check the second @@ -9044,31 +9133,31 @@ simdjson_really_inline void mini_formatter::number(double x) { buffer.insert(buffer.end(), number_buffer, newp); } -simdjson_really_inline void mini_formatter::start_array() { one_char('['); } -simdjson_really_inline void mini_formatter::end_array() { one_char(']'); } -simdjson_really_inline void mini_formatter::start_object() { one_char('{'); } -simdjson_really_inline void mini_formatter::end_object() { one_char('}'); } -simdjson_really_inline void mini_formatter::comma() { one_char(','); } +simdjson_inline void mini_formatter::start_array() { one_char('['); } +simdjson_inline void mini_formatter::end_array() { one_char(']'); } +simdjson_inline void mini_formatter::start_object() { one_char('{'); } +simdjson_inline void mini_formatter::end_object() { one_char('}'); } +simdjson_inline void mini_formatter::comma() { one_char(','); } -simdjson_really_inline void mini_formatter::true_atom() { +simdjson_inline void mini_formatter::true_atom() { const char * s = "true"; buffer.insert(buffer.end(), s, s + 4); } -simdjson_really_inline void mini_formatter::false_atom() { +simdjson_inline void mini_formatter::false_atom() { const char * s = "false"; buffer.insert(buffer.end(), s, s + 5); } -simdjson_really_inline void mini_formatter::null_atom() { +simdjson_inline void mini_formatter::null_atom() { const char * s = "null"; buffer.insert(buffer.end(), s, s + 4); } -simdjson_really_inline void mini_formatter::one_char(char c) { buffer.push_back(c); } -simdjson_really_inline void mini_formatter::key(std::string_view unescaped) { +simdjson_inline void mini_formatter::one_char(char c) { buffer.push_back(c); } +simdjson_inline void mini_formatter::key(std::string_view unescaped) { string(unescaped); one_char(':'); } -simdjson_really_inline void mini_formatter::string(std::string_view unescaped) { +simdjson_inline void mini_formatter::string(std::string_view unescaped) { one_char('\"'); size_t i = 0; // Fast path for the case where we have no control character, no ", and no backslash. @@ -9151,7 +9240,7 @@ inline void mini_formatter::clear() { buffer.clear(); } -simdjson_really_inline std::string_view mini_formatter::str() const { +simdjson_inline std::string_view mini_formatter::str() const { return std::string_view(buffer.data(), buffer.size()); } @@ -9322,18 +9411,18 @@ inline void string_builder::append(simdjson::dom::array value) { } template -simdjson_really_inline void string_builder::append(simdjson::dom::key_value_pair kv) { +simdjson_inline void string_builder::append(simdjson::dom::key_value_pair kv) { format.key(kv.key); append(kv.value); } template -simdjson_really_inline void string_builder::clear() { +simdjson_inline void string_builder::clear() { format.clear(); } template -simdjson_really_inline std::string_view string_builder::str() const { +simdjson_inline std::string_view string_builder::str() const { return format.str(); } @@ -9592,7 +9681,7 @@ using namespace simdjson::dom; class implementation final : public simdjson::implementation { public: - simdjson_really_inline implementation() : simdjson::implementation("arm64", "ARM NEON", internal::instruction_set::NEON) {} + simdjson_inline implementation() : simdjson::implementation("arm64", "ARM NEON", internal::instruction_set::NEON) {} simdjson_warn_unused error_code create_dom_parser_implementation( size_t capacity, size_t max_length, @@ -9650,10 +9739,11 @@ class dom_parser_implementation final : public internal::dom_parser_implementati simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: - simdjson_really_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); }; @@ -9702,6 +9792,8 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth( // you use visual studio or other compilers. #include +static_assert(sizeof(uint8x16_t) <= simdjson::SIMDJSON_PADDING, "insufficient padding for arm64"); + #endif // SIMDJSON_ARM64_INTRINSICS_H /* end file include/simdjson/arm64/intrinsics.h */ /* begin file include/simdjson/arm64/bitmanipulation.h */ @@ -9716,7 +9808,7 @@ namespace { // but the algorithms do not end up using the returned value. // Sadly, sanitizers are not smart enough to figure it out. SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_really_inline int trailing_zeroes(uint64_t input_num) { +simdjson_inline int trailing_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long ret; // Search the mask data from least significant bit (LSB) @@ -9729,12 +9821,12 @@ simdjson_really_inline int trailing_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { +simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { return input_num & (input_num-1); } /* result might be undefined when input_num is zero */ -simdjson_really_inline int leading_zeroes(uint64_t input_num) { +simdjson_inline int leading_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long leading_zero = 0; // Search the mask data from most significant bit (MSB) @@ -9749,7 +9841,7 @@ simdjson_really_inline int leading_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -simdjson_really_inline int count_ones(uint64_t input_num) { +simdjson_inline int count_ones(uint64_t input_num) { return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); } @@ -9768,7 +9860,7 @@ simdjson_really_inline int count_ones(uint64_t input_num) { #define SIMDJSON_PREFER_REVERSE_BITS 1 /* reverse the bits */ -simdjson_really_inline uint64_t reverse_bits(uint64_t input_num) { +simdjson_inline uint64_t reverse_bits(uint64_t input_num) { uint64_t rev_bits; __asm("rbit %0, %1" : "=r"(rev_bits) : "r"(input_num)); return rev_bits; @@ -9781,13 +9873,13 @@ simdjson_really_inline uint64_t reverse_bits(uint64_t input_num) { * of such undefined behavior is never used. **/ SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_really_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) { +simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) { return rev_bits ^ (uint64_t(0x8000000000000000) >> leading_zeroes); } #endif -simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { +simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO *result = value1 + value2; return *result < value1; @@ -9816,7 +9908,7 @@ namespace { // // For example, prefix_xor(00100100) == 00011100 // -simdjson_really_inline uint64_t prefix_xor(uint64_t bitmask) { +simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { ///////////// // We could do this with PMULL, but it is apparently slow. // @@ -9873,7 +9965,7 @@ namespace { * You should not use this function except for compile-time constants: * it is not efficient. */ -simdjson_really_inline uint8x16_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, +simdjson_inline uint8x16_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7, uint8_t x8, uint8_t x9, uint8_t x10, uint8_t x11, uint8_t x12, uint8_t x13, uint8_t x14, uint8_t x15, uint8_t x16) { @@ -9902,7 +9994,7 @@ simdjson_really_inline uint8x16_t make_uint8x16_t(uint8_t x1, uint8_t x2, uint return x; } -simdjson_really_inline uint8x8_t make_uint8x8_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, +simdjson_inline uint8x8_t make_uint8x8_t(uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6, uint8_t x7, uint8_t x8) { uint8x8_t x{}; x = vset_lane_u8(x1, x, 0); @@ -9917,7 +10009,7 @@ simdjson_really_inline uint8x8_t make_uint8x8_t(uint8_t x1, uint8_t x2, uint8_ } // We have to do the same work for make_int8x16_t -simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_t x4, +simdjson_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_t x4, int8_t x5, int8_t x6, int8_t x7, int8_t x8, int8_t x9, int8_t x10, int8_t x11, int8_t x12, int8_t x13, int8_t x14, int8_t x15, int8_t x16) { @@ -9963,24 +10055,24 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x static const int SIZE = sizeof(value); // Conversion from/to SIMD register - simdjson_really_inline base_u8(const uint8x16_t _value) : value(_value) {} - simdjson_really_inline operator const uint8x16_t&() const { return this->value; } - simdjson_really_inline operator uint8x16_t&() { return this->value; } + simdjson_inline base_u8(const uint8x16_t _value) : value(_value) {} + simdjson_inline operator const uint8x16_t&() const { return this->value; } + simdjson_inline operator uint8x16_t&() { return this->value; } // Bit operations - simdjson_really_inline simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } - simdjson_really_inline simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } - simdjson_really_inline simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } - simdjson_really_inline simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } - simdjson_really_inline simd8 operator~() const { return *this ^ 0xFFu; } - simdjson_really_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_really_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_really_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_inline simd8 operator|(const simd8 other) const { return vorrq_u8(*this, other); } + simdjson_inline simd8 operator&(const simd8 other) const { return vandq_u8(*this, other); } + simdjson_inline simd8 operator^(const simd8 other) const { return veorq_u8(*this, other); } + simdjson_inline simd8 bit_andnot(const simd8 other) const { return vbicq_u8(*this, other); } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8& operator|=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline simd8& operator&=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline simd8& operator^=(const simd8 other) { auto this_cast = static_cast*>(this); *this_cast = *this_cast ^ other; return *this_cast; } - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return vceqq_u8(lhs, rhs); } + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return vceqq_u8(lhs, rhs); } template - simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { return vextq_u8(prev_chunk, *this, 16 - N); } }; @@ -9991,17 +10083,17 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x typedef uint16_t bitmask_t; typedef uint32_t bitmask2_t; - static simdjson_really_inline simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } + static simdjson_inline simd8 splat(bool _value) { return vmovq_n_u8(uint8_t(-(!!_value))); } - simdjson_really_inline simd8(const uint8x16_t _value) : base_u8(_value) {} + simdjson_inline simd8(const uint8x16_t _value) : base_u8(_value) {} // False constructor - simdjson_really_inline simd8() : simd8(vdupq_n_u8(0)) {} + simdjson_inline simd8() : simd8(vdupq_n_u8(0)) {} // Splat constructor - simdjson_really_inline simd8(bool _value) : simd8(splat(_value)) {} + simdjson_inline simd8(bool _value) : simd8(splat(_value)) {} // We return uint32_t instead of uint16_t because that seems to be more efficient for most // purposes (cutting it down to uint16_t costs performance in some compilers). - simdjson_really_inline uint32_t to_bitmask() const { + simdjson_inline uint32_t to_bitmask() const { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO const uint8x16_t bit_mask = make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80); @@ -10015,26 +10107,26 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } - simdjson_really_inline bool any() const { return vmaxvq_u8(*this) != 0; } + simdjson_inline bool any() const { return vmaxvq_u8(*this) != 0; } }; // Unsigned bytes template<> struct simd8: base_u8 { - static simdjson_really_inline uint8x16_t splat(uint8_t _value) { return vmovq_n_u8(_value); } - static simdjson_really_inline uint8x16_t zero() { return vdupq_n_u8(0); } - static simdjson_really_inline uint8x16_t load(const uint8_t* values) { return vld1q_u8(values); } + static simdjson_inline uint8x16_t splat(uint8_t _value) { return vmovq_n_u8(_value); } + static simdjson_inline uint8x16_t zero() { return vdupq_n_u8(0); } + static simdjson_inline uint8x16_t load(const uint8_t* values) { return vld1q_u8(values); } - simdjson_really_inline simd8(const uint8x16_t _value) : base_u8(_value) {} + simdjson_inline simd8(const uint8x16_t _value) : base_u8(_value) {} // Zero constructor - simdjson_really_inline simd8() : simd8(zero()) {} + simdjson_inline simd8() : simd8(zero()) {} // Array constructor - simdjson_really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} // Splat constructor - simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Member-by-member initialization #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_really_inline simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) : simd8(make_uint8x16_t( @@ -10042,7 +10134,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x v8, v9, v10,v11,v12,v13,v14,v15 )) {} #else - simdjson_really_inline simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) : simd8(uint8x16_t{ @@ -10052,7 +10144,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x #endif // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -10063,44 +10155,44 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x } // Store to array - simdjson_really_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } + simdjson_inline void store(uint8_t dst[16]) const { return vst1q_u8(dst, *this); } // Saturated math - simdjson_really_inline simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } - simdjson_really_inline simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } + simdjson_inline simd8 saturating_add(const simd8 other) const { return vqaddq_u8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return vqsubq_u8(*this, other); } // Addition/subtraction are the same for signed and unsigned - simdjson_really_inline simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } - simdjson_really_inline simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } - simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } + simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_u8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_u8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } // Order-specific operations - simdjson_really_inline uint8_t max_val() const { return vmaxvq_u8(*this); } - simdjson_really_inline uint8_t min_val() const { return vminvq_u8(*this); } - simdjson_really_inline simd8 max_val(const simd8 other) const { return vmaxq_u8(*this, other); } - simdjson_really_inline simd8 min_val(const simd8 other) const { return vminq_u8(*this, other); } - simdjson_really_inline simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } - simdjson_really_inline simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } - simdjson_really_inline simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } - simdjson_really_inline simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } + simdjson_inline uint8_t max_val() const { return vmaxvq_u8(*this); } + simdjson_inline uint8_t min_val() const { return vminvq_u8(*this); } + simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_u8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return vminq_u8(*this, other); } + simdjson_inline simd8 operator<=(const simd8 other) const { return vcleq_u8(*this, other); } + simdjson_inline simd8 operator>=(const simd8 other) const { return vcgeq_u8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_u8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_u8(*this, other); } // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_really_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } + simdjson_inline simd8 gt_bits(const simd8 other) const { return simd8(*this > other); } // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true = nonzero. For ARM, returns all 1's. - simdjson_really_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } + simdjson_inline simd8 lt_bits(const simd8 other) const { return simd8(*this < other); } // Bit-specific operations - simdjson_really_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } - simdjson_really_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; } - simdjson_really_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return vtstq_u8(*this, bits); } + simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return (*this & bits).any_bits_set_anywhere(); } template - simdjson_really_inline simd8 shr() const { return vshrq_n_u8(*this, N); } + simdjson_inline simd8 shr() const { return vshrq_n_u8(*this, N); } template - simdjson_really_inline simd8 shl() const { return vshlq_n_u8(*this, N); } + simdjson_inline simd8 shl() const { return vshlq_n_u8(*this, N); } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) template - simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { return lookup_table.apply_lookup_16_to(*this); } @@ -10113,7 +10205,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x // signature simd8 compress(uint16_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_really_inline void compress(uint16_t mask, L * output) const { + simdjson_inline void compress(uint16_t mask, L * output) const { using internal::thintable_epi8; using internal::BitsSetTable256mul2; using internal::pshufb_combine_table; @@ -10150,7 +10242,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x // Copies all bytes corresponding to a 0 in the low half of the mask (interpreted as a // bitset) to output1, then those corresponding to a 0 in the high half to output2. template - simdjson_really_inline void compress_halves(uint16_t mask, L *output1, L *output2) const { + simdjson_inline void compress_halves(uint16_t mask, L *output1, L *output2) const { using internal::thintable_epi8; uint8_t mask1 = uint8_t(mask); // least significant 8 bits uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits @@ -10169,7 +10261,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x } template - simdjson_really_inline simd8 lookup_16( + simdjson_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -10183,7 +10275,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x } template - simdjson_really_inline simd8 apply_lookup_16_to(const simd8 original) { + simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { return vqtbl1q_u8(*this, simd8(original)); } }; @@ -10193,24 +10285,24 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x struct simd8 { int8x16_t value; - static simdjson_really_inline simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } - static simdjson_really_inline simd8 zero() { return vdupq_n_s8(0); } - static simdjson_really_inline simd8 load(const int8_t values[16]) { return vld1q_s8(values); } + static simdjson_inline simd8 splat(int8_t _value) { return vmovq_n_s8(_value); } + static simdjson_inline simd8 zero() { return vdupq_n_s8(0); } + static simdjson_inline simd8 load(const int8_t values[16]) { return vld1q_s8(values); } // Conversion from/to SIMD register - simdjson_really_inline simd8(const int8x16_t _value) : value{_value} {} - simdjson_really_inline operator const int8x16_t&() const { return this->value; } - simdjson_really_inline operator int8x16_t&() { return this->value; } + simdjson_inline simd8(const int8x16_t _value) : value{_value} {} + simdjson_inline operator const int8x16_t&() const { return this->value; } + simdjson_inline operator int8x16_t&() { return this->value; } // Zero constructor - simdjson_really_inline simd8() : simd8(zero()) {} + simdjson_inline simd8() : simd8(zero()) {} // Splat constructor - simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_really_inline simd8(const int8_t* values) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} // Member-by-member initialization #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_really_inline simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) : simd8(make_int8x16_t( @@ -10218,7 +10310,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x v8, v9, v10,v11,v12,v13,v14,v15 )) {} #else - simdjson_really_inline simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) : simd8(int8x16_t{ @@ -10227,7 +10319,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x }) {} #endif // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -10238,7 +10330,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x } // Store to array - simdjson_really_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, *this); } + simdjson_inline void store(int8_t dst[16]) const { return vst1q_s8(dst, *this); } // Explicit conversion to/from unsigned // @@ -10246,35 +10338,35 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x // In theory, we could check this occurrence with std::same_as and std::enabled_if but it is C++14 // and relatively ugly and hard to read. #ifndef SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_really_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {} + simdjson_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {} #endif - simdjson_really_inline explicit operator simd8() const { return vreinterpretq_u8_s8(this->value); } + simdjson_inline explicit operator simd8() const { return vreinterpretq_u8_s8(this->value); } // Math - simdjson_really_inline simd8 operator+(const simd8 other) const { return vaddq_s8(*this, other); } - simdjson_really_inline simd8 operator-(const simd8 other) const { return vsubq_s8(*this, other); } - simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } - simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } + simdjson_inline simd8 operator+(const simd8 other) const { return vaddq_s8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return vsubq_s8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *this; } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *this; } // Order-sensitive comparisons - simdjson_really_inline simd8 max_val(const simd8 other) const { return vmaxq_s8(*this, other); } - simdjson_really_inline simd8 min_val(const simd8 other) const { return vminq_s8(*this, other); } - simdjson_really_inline simd8 operator>(const simd8 other) const { return vcgtq_s8(*this, other); } - simdjson_really_inline simd8 operator<(const simd8 other) const { return vcltq_s8(*this, other); } - simdjson_really_inline simd8 operator==(const simd8 other) const { return vceqq_s8(*this, other); } + simdjson_inline simd8 max_val(const simd8 other) const { return vmaxq_s8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return vminq_s8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return vcgtq_s8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return vcltq_s8(*this, other); } + simdjson_inline simd8 operator==(const simd8 other) const { return vceqq_s8(*this, other); } template - simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { return vextq_s8(prev_chunk, *this, 16 - N); } // Perform a lookup assuming no value is larger than 16 template - simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { return lookup_table.apply_lookup_16_to(*this); } template - simdjson_really_inline simd8 lookup_16( + simdjson_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -10288,7 +10380,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x } template - simdjson_really_inline simd8 apply_lookup_16_to(const simd8 original) { + simdjson_inline simd8 apply_lookup_16_to(const simd8 original) { return vqtbl1q_s8(*this, simd8(original)); } }; @@ -10303,22 +10395,22 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x simd8x64& operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - simdjson_really_inline void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); this->chunks[1].store(ptr+sizeof(simd8)*1); this->chunks[2].store(ptr+sizeof(simd8)*2); this->chunks[3].store(ptr+sizeof(simd8)*3); } - simdjson_really_inline simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); } - simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const { + simdjson_inline uint64_t compress(uint64_t mask, T * output) const { uint64_t popcounts = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vcreate_u8(~mask))), 0); // compute the prefix sum of the popcounts of each byte uint64_t offsets = popcounts * 0x0101010101010101; @@ -10329,7 +10421,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x return offsets >> 56; } - simdjson_really_inline uint64_t to_bitmask() const { + simdjson_inline uint64_t to_bitmask() const { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO const uint8x16_t bit_mask = make_uint8x16_t( 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, @@ -10349,7 +10441,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0); } - simdjson_really_inline uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] == mask, @@ -10359,7 +10451,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x ).to_bitmask(); } - simdjson_really_inline uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] <= mask, @@ -10386,11 +10478,11 @@ namespace jsoncharutils { // return non-zero if not a structural or whitespace char // zero otherwise -simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace_negated[c]; } -simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace[c]; } @@ -10421,7 +10513,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -10453,10 +10545,10 @@ simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -10470,7 +10562,7 @@ static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64 using internal::value128; -simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -10506,13 +10598,13 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. simdjson_warn_unused -simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); @@ -10520,36 +10612,36 @@ simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -10575,18 +10667,18 @@ using namespace simd; struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_really_inline bool has_backslash() { return bs_bits != 0; } - simdjson_really_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } uint32_t bs_bits; uint32_t quote_bits; }; // struct backslash_and_quote -simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { +simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); @@ -10608,147 +10700,6 @@ simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(co } // namespace arm64 } // namespace simdjson -/* begin file include/simdjson/generic/stringparsing.h */ -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace arm64 { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // check for low surrogate for characters outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - - // if the first code point is invalid we will get here, as we will go past - // the check for being outside the Basic Multilingual plane. If we don't - // find a \u immediately afterwards we fail out anyhow, but if we do, - // this check catches both the case of the first code point being invalid - // or the second code point being invalid. - if ((code_point | code_point_2) >> 16) { - return false; - } - - code_point = - (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000; - *src_ptr += 6; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - -/** - * Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then - * dst needs to have four free bytes. - */ -simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - /* can't be reached */ - return nullptr; -} - -simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { - if (*(src++) != '"') { return STRING_ERROR; } - auto end = stringparsing::parse_string(src, current_string_buf_loc); - if (!end) { return STRING_ERROR; } - s = std::string_view(reinterpret_cast(current_string_buf_loc), end-current_string_buf_loc); - current_string_buf_loc = end; - return SUCCESS; -} - -} // namespace stringparsing -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson -/* end file include/simdjson/generic/stringparsing.h */ - #endif // SIMDJSON_ARM64_STRINGPARSING_H /* end file include/simdjson/arm64/stringparsing.h */ /* begin file include/simdjson/arm64/numberparsing.h */ @@ -10761,7 +10712,7 @@ namespace { // we don't have SSE, so let us use a scalar function // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { +static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { uint64_t val; std::memcpy(&val, chars, sizeof(uint64_t)); val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; @@ -10814,7 +10765,7 @@ namespace { // Convert a mantissa, an exponent and a sign bit into an ieee64 double. // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { +simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { double d; mantissa &= ~(1ULL << 52); mantissa |= real_exponent << 52; @@ -10829,7 +10780,7 @@ simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponen // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [smallest_power, // largest_power] interval: the caller is responsible for this check. -simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -10889,7 +10840,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg // In the slow path, we need to adjust i so that it is > 1<<63 which is always // possible, except if i == 0, so we handle i == 0 separately. if(i == 0) { - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } @@ -11004,7 +10955,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? // Here have that real_exponent <= 0 so -real_exponent >= 0 if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } // next line is safe because -real_exponent + 1 < 0 @@ -11112,7 +11063,7 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -11139,7 +11090,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) { template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -11149,7 +11100,7 @@ simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -11177,7 +11128,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c return SUCCESS; } -simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -11228,7 +11179,7 @@ simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t * return SUCCESS; } -simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { +simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -11238,7 +11189,7 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s } template -simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { +simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. @@ -11274,7 +11225,8 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); // if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - WRITE_DOUBLE(0, src, writer); + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); return SUCCESS; } else { // (exponent > largest_power) and (i != 0) // We have, for sure, an infinite value and simdjson refuses to parse infinite values. @@ -11294,20 +11246,20 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg #ifdef SIMDJSON_SKIPNUMBERPARSING template -simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } #else // parse the number at src @@ -11320,13 +11272,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge // // Our objective is accurate parsing (ULP of 0) at high speed. template -simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -11470,7 +11422,7 @@ const uint8_t integer_string_finisher[256] = { NUMBER_ERROR}; // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -11520,7 +11472,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( // Parse any number from 0 to 18,446,744,073,709,551,615 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -11568,7 +11520,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( } // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { const uint8_t *p = src + 1; // // Parse the integer part. @@ -11618,12 +11570,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -11661,13 +11613,13 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { // // Check for minus sign // if(src == src_end) { return NUMBER_ERROR; } bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -11704,24 +11656,24 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - const uint8_t *p = src + negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. // // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; + const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + while (parse_digit(*src, i)) { src++; } // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); + size_t digit_count = size_t(src - start_digits); // We go from // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // so we can never represent numbers that have more than 19 digits. @@ -11733,11 +11685,11 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in // Here digit_count > 0. if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; // } // as a single table lookup: - if(*p != '"') { return NUMBER_ERROR; } + if(*src != '"') { return NUMBER_ERROR; } // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. // Performance note: This check is only needed when digit_count == longest_digit_count but it is // so cheap that we might as well always make it. @@ -11745,12 +11697,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in return negative ? (~i+1) : i; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -11817,19 +11769,19 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -11837,9 +11789,9 @@ simdjson_unused simdjson_really_inline simdjson_result is_integer(const ui return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -11863,13 +11815,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge } // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { if(src == src_end) { return NUMBER_ERROR; } // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -11938,18 +11890,18 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, src_end, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - src += negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. @@ -12016,7 +11968,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double_in_s if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; @@ -12069,7 +12021,7 @@ using namespace simdjson::dom; class implementation final : public simdjson::implementation { public: - simdjson_really_inline implementation() : simdjson::implementation( + simdjson_inline implementation() : simdjson::implementation( "fallback", "Generic fallback implementation", 0 @@ -12131,10 +12083,11 @@ class dom_parser_implementation final : public internal::dom_parser_implementati simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: - simdjson_really_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); }; @@ -12203,7 +12156,7 @@ static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) { #endif /* result might be undefined when input_num is zero */ -simdjson_really_inline int leading_zeroes(uint64_t input_num) { +simdjson_inline int leading_zeroes(uint64_t input_num) { #ifdef _MSC_VER unsigned long leading_zero = 0; // Search the mask data from most significant bit (MSB) @@ -12232,11 +12185,11 @@ namespace jsoncharutils { // return non-zero if not a structural or whitespace char // zero otherwise -simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace_negated[c]; } -simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace[c]; } @@ -12267,7 +12220,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -12299,10 +12252,10 @@ simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -12316,7 +12269,7 @@ static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64 using internal::value128; -simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -12352,13 +12305,13 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. simdjson_warn_unused -simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); @@ -12366,36 +12319,36 @@ simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -12419,17 +12372,17 @@ namespace { struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 1; - simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - simdjson_really_inline bool has_quote_first() { return c == '"'; } - simdjson_really_inline bool has_backslash() { return c == '\\'; } - simdjson_really_inline int quote_index() { return c == '"' ? 0 : 1; } - simdjson_really_inline int backslash_index() { return c == '\\' ? 0 : 1; } + simdjson_inline bool has_quote_first() { return c == '"'; } + simdjson_inline bool has_backslash() { return c == '\\'; } + simdjson_inline int quote_index() { return c == '"' ? 0 : 1; } + simdjson_inline int backslash_index() { return c == '\\' ? 0 : 1; } uint8_t c; }; // struct backslash_and_quote -simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { +simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // store to dest unconditionally - we can overwrite the bits we don't like later dst[0] = src[0]; return { src[0] }; @@ -12439,147 +12392,6 @@ simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(co } // namespace fallback } // namespace simdjson -/* begin file include/simdjson/generic/stringparsing.h */ -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace fallback { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // check for low surrogate for characters outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - - // if the first code point is invalid we will get here, as we will go past - // the check for being outside the Basic Multilingual plane. If we don't - // find a \u immediately afterwards we fail out anyhow, but if we do, - // this check catches both the case of the first code point being invalid - // or the second code point being invalid. - if ((code_point | code_point_2) >> 16) { - return false; - } - - code_point = - (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000; - *src_ptr += 6; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - -/** - * Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then - * dst needs to have four free bytes. - */ -simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - /* can't be reached */ - return nullptr; -} - -simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { - if (*(src++) != '"') { return STRING_ERROR; } - auto end = stringparsing::parse_string(src, current_string_buf_loc); - if (!end) { return STRING_ERROR; } - s = std::string_view(reinterpret_cast(current_string_buf_loc), end-current_string_buf_loc); - current_string_buf_loc = end; - return SUCCESS; -} - -} // namespace stringparsing -} // unnamed namespace -} // namespace fallback -} // namespace simdjson -/* end file include/simdjson/generic/stringparsing.h */ - #endif // SIMDJSON_FALLBACK_STRINGPARSING_H /* end file include/simdjson/fallback/stringparsing.h */ /* begin file include/simdjson/fallback/numberparsing.h */ @@ -12597,14 +12409,14 @@ namespace simdjson { namespace fallback { namespace { // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const char *chars) { +static simdjson_inline uint32_t parse_eight_digits_unrolled(const char *chars) { uint64_t val; memcpy(&val, chars, sizeof(uint64_t)); val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); } -static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { +static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { return parse_eight_digits_unrolled(reinterpret_cast(chars)); } @@ -12653,7 +12465,7 @@ namespace { // Convert a mantissa, an exponent and a sign bit into an ieee64 double. // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { +simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { double d; mantissa &= ~(1ULL << 52); mantissa |= real_exponent << 52; @@ -12668,7 +12480,7 @@ simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponen // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [smallest_power, // largest_power] interval: the caller is responsible for this check. -simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -12728,7 +12540,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg // In the slow path, we need to adjust i so that it is > 1<<63 which is always // possible, except if i == 0, so we handle i == 0 separately. if(i == 0) { - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } @@ -12843,7 +12655,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? // Here have that real_exponent <= 0 so -real_exponent >= 0 if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } // next line is safe because -real_exponent + 1 < 0 @@ -12951,7 +12763,7 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -12978,7 +12790,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) { template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -12988,7 +12800,7 @@ simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -13016,7 +12828,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c return SUCCESS; } -simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -13067,7 +12879,7 @@ simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t * return SUCCESS; } -simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { +simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -13077,7 +12889,7 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s } template -simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { +simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. @@ -13113,7 +12925,8 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); // if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - WRITE_DOUBLE(0, src, writer); + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); return SUCCESS; } else { // (exponent > largest_power) and (i != 0) // We have, for sure, an infinite value and simdjson refuses to parse infinite values. @@ -13133,20 +12946,20 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg #ifdef SIMDJSON_SKIPNUMBERPARSING template -simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } #else // parse the number at src @@ -13159,13 +12972,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge // // Our objective is accurate parsing (ULP of 0) at high speed. template -simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -13309,7 +13122,7 @@ const uint8_t integer_string_finisher[256] = { NUMBER_ERROR}; // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -13359,7 +13172,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( // Parse any number from 0 to 18,446,744,073,709,551,615 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -13407,7 +13220,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( } // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { const uint8_t *p = src + 1; // // Parse the integer part. @@ -13457,12 +13270,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -13500,13 +13313,13 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { // // Check for minus sign // if(src == src_end) { return NUMBER_ERROR; } bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -13543,24 +13356,24 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - const uint8_t *p = src + negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. // // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; + const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + while (parse_digit(*src, i)) { src++; } // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); + size_t digit_count = size_t(src - start_digits); // We go from // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // so we can never represent numbers that have more than 19 digits. @@ -13572,11 +13385,11 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in // Here digit_count > 0. if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; // } // as a single table lookup: - if(*p != '"') { return NUMBER_ERROR; } + if(*src != '"') { return NUMBER_ERROR; } // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. // Performance note: This check is only needed when digit_count == longest_digit_count but it is // so cheap that we might as well always make it. @@ -13584,12 +13397,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in return negative ? (~i+1) : i; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -13656,19 +13469,19 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -13676,9 +13489,9 @@ simdjson_unused simdjson_really_inline simdjson_result is_integer(const ui return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -13702,13 +13515,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge } // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { if(src == src_end) { return NUMBER_ERROR; } // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -13777,18 +13590,18 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, src_end, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - src += negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. @@ -13855,7 +13668,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double_in_s if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; @@ -13916,7 +13729,7 @@ using namespace simdjson; class implementation final : public simdjson::implementation { public: - simdjson_really_inline implementation() : simdjson::implementation( + simdjson_inline implementation() : simdjson::implementation( "icelake", "Intel/AMD AVX512", internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 | internal::instruction_set::AVX512F | internal::instruction_set::AVX512DQ | internal::instruction_set::AVX512CD | internal::instruction_set::AVX512BW | internal::instruction_set::AVX512VL | internal::instruction_set::AVX512VBMI2 @@ -13990,6 +13803,8 @@ class implementation final : public simdjson::implementation { #endif // _blsr_u64 #endif // SIMDJSON_CLANG_VISUAL_STUDIO +static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); + #endif // SIMDJSON_ICELAKE_INTRINSICS_H /* end file include/simdjson/icelake/intrinsics.h */ @@ -14039,10 +13854,11 @@ class dom_parser_implementation final : public internal::dom_parser_implementati simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: - simdjson_really_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); }; @@ -14095,7 +13911,7 @@ namespace { // but the algorithms do not end up using the returned value. // Sadly, sanitizers are not smart enough to figure it out. SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_really_inline int trailing_zeroes(uint64_t input_num) { +simdjson_inline int trailing_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO return (int)_tzcnt_u64(input_num); #else // SIMDJSON_REGULAR_VISUAL_STUDIO @@ -14109,27 +13925,27 @@ simdjson_really_inline int trailing_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { +simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { return _blsr_u64(input_num); } /* result might be undefined when input_num is zero */ -simdjson_really_inline int leading_zeroes(uint64_t input_num) { +simdjson_inline int leading_zeroes(uint64_t input_num) { return int(_lzcnt_u64(input_num)); } #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_really_inline unsigned __int64 count_ones(uint64_t input_num) { +simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { // note: we do not support legacy 32-bit Windows return __popcnt64(input_num);// Visual Studio wants two underscores } #else -simdjson_really_inline long long int count_ones(uint64_t input_num) { +simdjson_inline long long int count_ones(uint64_t input_num) { return _popcnt64(input_num); } #endif -simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, +simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO return _addcarry_u64(0, value1, value2, @@ -14159,7 +13975,7 @@ namespace { // // For example, prefix_xor(00100100) == 00011100 // -simdjson_really_inline uint64_t prefix_xor(const uint64_t bitmask) { +simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { // There should be no such thing with a processor supporting avx2 // but not clmul. __m128i all_ones = _mm_set1_epi8('\xFF'); @@ -14215,23 +14031,23 @@ namespace simd { __m512i value; // Zero constructor - simdjson_really_inline base() : value{__m512i()} {} + simdjson_inline base() : value{__m512i()} {} // Conversion from SIMD register - simdjson_really_inline base(const __m512i _value) : value(_value) {} + simdjson_inline base(const __m512i _value) : value(_value) {} // Conversion to SIMD register - simdjson_really_inline operator const __m512i&() const { return this->value; } - simdjson_really_inline operator __m512i&() { return this->value; } + simdjson_inline operator const __m512i&() const { return this->value; } + simdjson_inline operator __m512i&() { return this->value; } // Bit operations - simdjson_really_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } - simdjson_really_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } - simdjson_really_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } - simdjson_really_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } - simdjson_really_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_really_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_really_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } }; // Forward-declared so they can be used by splat and friends. @@ -14243,15 +14059,17 @@ namespace simd { typedef uint32_t bitmask_t; typedef uint64_t bitmask2_t; - simdjson_really_inline base8() : base>() {} - simdjson_really_inline base8(const __m512i _value) : base>(_value) {} + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m512i _value) : base>(_value) {} - simdjson_really_inline uint64_t operator==(const simd8 other) const { return _mm512_cmpeq_epi8_mask(*this, other); } + friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { + return _mm512_cmpeq_epi8_mask(lhs, rhs); + } static const int SIZE = sizeof(base::value); template - simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { #if SIMDJSON_GCC8 // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8) constexpr int shift = 16 - N; @@ -14265,25 +14083,25 @@ namespace simd { // SIMD byte mask type (returned by things like eq and gt) template<> struct simd8: base8 { - static simdjson_really_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } + static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } - simdjson_really_inline simd8() : base8() {} - simdjson_really_inline simd8(const __m512i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m512i _value) : base8(_value) {} // Splat constructor - simdjson_really_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_really_inline bool any() const { return !!_mm512_test_epi8_mask (*this, *this); } - simdjson_really_inline simd8 operator~() const { return *this ^ true; } + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline bool any() const { return !!_mm512_test_epi8_mask (*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } }; template struct base8_numeric: base8 { - static simdjson_really_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } - static simdjson_really_inline simd8 zero() { return _mm512_setzero_si512(); } - static simdjson_really_inline simd8 load(const T values[64]) { + static simdjson_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm512_setzero_si512(); } + static simdjson_inline simd8 load(const T values[64]) { return _mm512_loadu_si512(reinterpret_cast(values)); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_really_inline simd8 repeat_16( + static simdjson_inline simd8 repeat_16( T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 ) { @@ -14299,24 +14117,24 @@ namespace simd { ); } - simdjson_really_inline base8_numeric() : base8() {} - simdjson_really_inline base8_numeric(const __m512i _value) : base8(_value) {} + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m512i _value) : base8(_value) {} // Store to array - simdjson_really_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i *>(dst), *this); } + simdjson_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i *>(dst), *this); } // Addition/subtraction are the same for signed and unsigned - simdjson_really_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } - simdjson_really_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } - simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + simdjson_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } // Override to distinguish from bool version - simdjson_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) template - simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { return _mm512_shuffle_epi8(lookup_table, *this); } @@ -14328,12 +14146,12 @@ namespace simd { // signature simd8 compress(uint32_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_really_inline void compress(uint64_t mask, L * output) const { + simdjson_inline void compress(uint64_t mask, L * output) const { _mm512_mask_compressstoreu_epi8 (output,~mask,*this); } template - simdjson_really_inline simd8 lookup_16( + simdjson_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -14350,14 +14168,14 @@ namespace simd { // Signed bytes template<> struct simd8 : base8_numeric { - simdjson_really_inline simd8() : base8_numeric() {} - simdjson_really_inline simd8(const __m512i _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} // Splat constructor - simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_really_inline simd8(const int8_t values[64]) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t values[64]) : simd8(load(values)) {} // Member-by-member initialization - simdjson_really_inline simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, @@ -14378,7 +14196,7 @@ namespace simd { )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -14395,24 +14213,24 @@ namespace simd { } // Order-sensitive comparisons - simdjson_really_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } - simdjson_really_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } - simdjson_really_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other),_mm512_set1_epi8(uint8_t(0x80))); } - simdjson_really_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this),_mm512_set1_epi8(uint8_t(0x80))); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other),_mm512_set1_epi8(uint8_t(0x80))); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this),_mm512_set1_epi8(uint8_t(0x80))); } }; // Unsigned bytes template<> struct simd8: base8_numeric { - simdjson_really_inline simd8() : base8_numeric() {} - simdjson_really_inline simd8(const __m512i _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} // Splat constructor - simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_really_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} // Member-by-member initialization - simdjson_really_inline simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, @@ -14433,7 +14251,7 @@ namespace simd { )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -14450,42 +14268,42 @@ namespace simd { } // Saturated math - simdjson_really_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } - simdjson_really_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } // Order-specific operations - simdjson_really_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } - simdjson_really_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_really_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_really_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_really_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } // Bit-specific operations - simdjson_really_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } - simdjson_really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_really_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } - simdjson_really_inline bool bits_not_set_anywhere() const { + simdjson_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return !_mm512_test_epi8_mask(*this, *this); } - simdjson_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_really_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } - simdjson_really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } template - simdjson_really_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + simdjson_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } template - simdjson_really_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + simdjson_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } // Get one of the bits and make a bitmask out of it. // e.g. value.get_bit<7>() gets the high bit template - simdjson_really_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); } + simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); } }; template @@ -14498,40 +14316,40 @@ namespace simd { simd8x64& operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_really_inline simd8x64(const simd8 chunk0) : chunks{chunk0} {} - simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr)} {} + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} + simdjson_inline simd8x64(const simd8 chunk0) : chunks{chunk0} {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr)} {} - simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const { + simdjson_inline uint64_t compress(uint64_t mask, T * output) const { this->chunks[0].compress(mask, output); return 64 - count_ones(mask); } - simdjson_really_inline void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); } - simdjson_really_inline simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return this->chunks[0]; } - simdjson_really_inline simd8x64 bit_or(const T m) const { + simdjson_inline simd8x64 bit_or(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] | mask ); } - simdjson_really_inline uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return this->chunks[0] == mask; } - simdjson_really_inline uint64_t eq(const simd8x64 &other) const { + simdjson_inline uint64_t eq(const simd8x64 &other) const { return this->chunks[0] == other.chunks[0]; } - simdjson_really_inline uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return this->chunks[0] <= mask; } @@ -14554,11 +14372,11 @@ namespace jsoncharutils { // return non-zero if not a structural or whitespace char // zero otherwise -simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace_negated[c]; } -simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace[c]; } @@ -14589,7 +14407,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -14621,10 +14439,10 @@ simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -14638,7 +14456,7 @@ static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64 using internal::value128; -simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -14674,13 +14492,13 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. simdjson_warn_unused -simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); @@ -14688,36 +14506,36 @@ simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -14743,18 +14561,18 @@ using namespace simd; struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_really_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } uint64_t bs_bits; uint64_t quote_bits; }; // struct backslash_and_quote -simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { +simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 15 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); @@ -14771,147 +14589,6 @@ simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(co } // namespace icelake } // namespace simdjson -/* begin file include/simdjson/generic/stringparsing.h */ -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace icelake { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // check for low surrogate for characters outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - - // if the first code point is invalid we will get here, as we will go past - // the check for being outside the Basic Multilingual plane. If we don't - // find a \u immediately afterwards we fail out anyhow, but if we do, - // this check catches both the case of the first code point being invalid - // or the second code point being invalid. - if ((code_point | code_point_2) >> 16) { - return false; - } - - code_point = - (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000; - *src_ptr += 6; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - -/** - * Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then - * dst needs to have four free bytes. - */ -simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - /* can't be reached */ - return nullptr; -} - -simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { - if (*(src++) != '"') { return STRING_ERROR; } - auto end = stringparsing::parse_string(src, current_string_buf_loc); - if (!end) { return STRING_ERROR; } - s = std::string_view(reinterpret_cast(current_string_buf_loc), end-current_string_buf_loc); - current_string_buf_loc = end; - return SUCCESS; -} - -} // namespace stringparsing -} // unnamed namespace -} // namespace icelake -} // namespace simdjson -/* end file include/simdjson/generic/stringparsing.h */ - #endif // SIMDJSON_ICELAKE_STRINGPARSING_H /* end file include/simdjson/icelake/stringparsing.h */ /* begin file include/simdjson/icelake/numberparsing.h */ @@ -14922,7 +14599,7 @@ namespace simdjson { namespace icelake { namespace { -static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { +static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); const __m128i mul_1_10 = @@ -14985,7 +14662,7 @@ namespace { // Convert a mantissa, an exponent and a sign bit into an ieee64 double. // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { +simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { double d; mantissa &= ~(1ULL << 52); mantissa |= real_exponent << 52; @@ -15000,7 +14677,7 @@ simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponen // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [smallest_power, // largest_power] interval: the caller is responsible for this check. -simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -15060,7 +14737,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg // In the slow path, we need to adjust i so that it is > 1<<63 which is always // possible, except if i == 0, so we handle i == 0 separately. if(i == 0) { - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } @@ -15175,7 +14852,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? // Here have that real_exponent <= 0 so -real_exponent >= 0 if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } // next line is safe because -real_exponent + 1 < 0 @@ -15283,7 +14960,7 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -15310,7 +14987,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) { template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -15320,7 +14997,7 @@ simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -15348,7 +15025,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c return SUCCESS; } -simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -15399,7 +15076,7 @@ simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t * return SUCCESS; } -simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { +simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -15409,7 +15086,7 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s } template -simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { +simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. @@ -15445,7 +15122,8 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); // if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - WRITE_DOUBLE(0, src, writer); + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); return SUCCESS; } else { // (exponent > largest_power) and (i != 0) // We have, for sure, an infinite value and simdjson refuses to parse infinite values. @@ -15465,20 +15143,20 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg #ifdef SIMDJSON_SKIPNUMBERPARSING template -simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } #else // parse the number at src @@ -15491,13 +15169,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge // // Our objective is accurate parsing (ULP of 0) at high speed. template -simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -15641,7 +15319,7 @@ const uint8_t integer_string_finisher[256] = { NUMBER_ERROR}; // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -15691,7 +15369,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( // Parse any number from 0 to 18,446,744,073,709,551,615 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -15739,7 +15417,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( } // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { const uint8_t *p = src + 1; // // Parse the integer part. @@ -15789,12 +15467,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -15832,13 +15510,13 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { // // Check for minus sign // if(src == src_end) { return NUMBER_ERROR; } bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -15875,24 +15553,24 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - const uint8_t *p = src + negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. // // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; + const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + while (parse_digit(*src, i)) { src++; } // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); + size_t digit_count = size_t(src - start_digits); // We go from // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // so we can never represent numbers that have more than 19 digits. @@ -15904,11 +15582,11 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in // Here digit_count > 0. if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; // } // as a single table lookup: - if(*p != '"') { return NUMBER_ERROR; } + if(*src != '"') { return NUMBER_ERROR; } // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. // Performance note: This check is only needed when digit_count == longest_digit_count but it is // so cheap that we might as well always make it. @@ -15916,12 +15594,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in return negative ? (~i+1) : i; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -15988,19 +15666,19 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -16008,9 +15686,9 @@ simdjson_unused simdjson_really_inline simdjson_result is_integer(const ui return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -16034,13 +15712,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge } // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { if(src == src_end) { return NUMBER_ERROR; } // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -16109,18 +15787,18 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, src_end, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - src += negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. @@ -16187,7 +15865,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double_in_s if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; @@ -16249,7 +15927,7 @@ using namespace simdjson; class implementation final : public simdjson::implementation { public: - simdjson_really_inline implementation() : simdjson::implementation( + simdjson_inline implementation() : simdjson::implementation( "haswell", "Intel/AMD AVX2", internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 @@ -16315,6 +15993,8 @@ class implementation final : public simdjson::implementation { #endif // _blsr_u64 #endif // SIMDJSON_CLANG_VISUAL_STUDIO +static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); + #endif // SIMDJSON_HASWELL_INTRINSICS_H /* end file include/simdjson/haswell/intrinsics.h */ @@ -16364,10 +16044,11 @@ class dom_parser_implementation final : public internal::dom_parser_implementati simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: - simdjson_really_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); }; @@ -16420,7 +16101,7 @@ namespace { // but the algorithms do not end up using the returned value. // Sadly, sanitizers are not smart enough to figure it out. SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_really_inline int trailing_zeroes(uint64_t input_num) { +simdjson_inline int trailing_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO return (int)_tzcnt_u64(input_num); #else // SIMDJSON_REGULAR_VISUAL_STUDIO @@ -16434,27 +16115,27 @@ simdjson_really_inline int trailing_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { +simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { return _blsr_u64(input_num); } /* result might be undefined when input_num is zero */ -simdjson_really_inline int leading_zeroes(uint64_t input_num) { +simdjson_inline int leading_zeroes(uint64_t input_num) { return int(_lzcnt_u64(input_num)); } #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_really_inline unsigned __int64 count_ones(uint64_t input_num) { +simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { // note: we do not support legacy 32-bit Windows return __popcnt64(input_num);// Visual Studio wants two underscores } #else -simdjson_really_inline long long int count_ones(uint64_t input_num) { +simdjson_inline long long int count_ones(uint64_t input_num) { return _popcnt64(input_num); } #endif -simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, +simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO return _addcarry_u64(0, value1, value2, @@ -16484,7 +16165,7 @@ namespace { // // For example, prefix_xor(00100100) == 00011100 // -simdjson_really_inline uint64_t prefix_xor(const uint64_t bitmask) { +simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { // There should be no such thing with a processor supporting avx2 // but not clmul. __m128i all_ones = _mm_set1_epi8('\xFF'); @@ -16514,23 +16195,23 @@ namespace simd { __m256i value; // Zero constructor - simdjson_really_inline base() : value{__m256i()} {} + simdjson_inline base() : value{__m256i()} {} // Conversion from SIMD register - simdjson_really_inline base(const __m256i _value) : value(_value) {} + simdjson_inline base(const __m256i _value) : value(_value) {} // Conversion to SIMD register - simdjson_really_inline operator const __m256i&() const { return this->value; } - simdjson_really_inline operator __m256i&() { return this->value; } + simdjson_inline operator const __m256i&() const { return this->value; } + simdjson_inline operator __m256i&() { return this->value; } // Bit operations - simdjson_really_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } - simdjson_really_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } - simdjson_really_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } - simdjson_really_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } - simdjson_really_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_really_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_really_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } }; // Forward-declared so they can be used by splat and friends. @@ -16542,15 +16223,15 @@ namespace simd { typedef uint32_t bitmask_t; typedef uint64_t bitmask2_t; - simdjson_really_inline base8() : base>() {} - simdjson_really_inline base8(const __m256i _value) : base>(_value) {} + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m256i _value) : base>(_value) {} friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm256_cmpeq_epi8(lhs, rhs); } static const int SIZE = sizeof(base::value); template - simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); } }; @@ -16558,27 +16239,27 @@ namespace simd { // SIMD byte mask type (returned by things like eq and gt) template<> struct simd8: base8 { - static simdjson_really_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } + static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - simdjson_really_inline simd8() : base8() {} - simdjson_really_inline simd8(const __m256i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m256i _value) : base8(_value) {} // Splat constructor - simdjson_really_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_really_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } - simdjson_really_inline bool any() const { return !_mm256_testz_si256(*this, *this); } - simdjson_really_inline simd8 operator~() const { return *this ^ true; } + simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } }; template struct base8_numeric: base8 { - static simdjson_really_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } - static simdjson_really_inline simd8 zero() { return _mm256_setzero_si256(); } - static simdjson_really_inline simd8 load(const T values[32]) { + static simdjson_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm256_setzero_si256(); } + static simdjson_inline simd8 load(const T values[32]) { return _mm256_loadu_si256(reinterpret_cast(values)); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_really_inline simd8 repeat_16( + static simdjson_inline simd8 repeat_16( T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 ) { @@ -16590,24 +16271,24 @@ namespace simd { ); } - simdjson_really_inline base8_numeric() : base8() {} - simdjson_really_inline base8_numeric(const __m256i _value) : base8(_value) {} + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} // Store to array - simdjson_really_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); } + simdjson_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); } // Addition/subtraction are the same for signed and unsigned - simdjson_really_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } - simdjson_really_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } - simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + simdjson_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } // Override to distinguish from bool version - simdjson_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) template - simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { return _mm256_shuffle_epi8(lookup_table, *this); } @@ -16619,7 +16300,7 @@ namespace simd { // signature simd8 compress(uint32_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_really_inline void compress(uint32_t mask, L * output) const { + simdjson_inline void compress(uint32_t mask, L * output) const { using internal::thintable_epi8; using internal::BitsSetTable256mul2; using internal::pshufb_combine_table; @@ -16665,7 +16346,7 @@ namespace simd { } template - simdjson_really_inline simd8 lookup_16( + simdjson_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -16682,14 +16363,14 @@ namespace simd { // Signed bytes template<> struct simd8 : base8_numeric { - simdjson_really_inline simd8() : base8_numeric() {} - simdjson_really_inline simd8(const __m256i _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} // Splat constructor - simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_really_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} // Member-by-member initialization - simdjson_really_inline simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, @@ -16701,7 +16382,7 @@ namespace simd { v24,v25,v26,v27,v28,v29,v30,v31 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -16714,23 +16395,23 @@ namespace simd { } // Order-sensitive comparisons - simdjson_really_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } - simdjson_really_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } - simdjson_really_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } - simdjson_really_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } }; // Unsigned bytes template<> struct simd8: base8_numeric { - simdjson_really_inline simd8() : base8_numeric() {} - simdjson_really_inline simd8(const __m256i _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} // Splat constructor - simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} // Member-by-member initialization - simdjson_really_inline simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, @@ -16742,7 +16423,7 @@ namespace simd { v24,v25,v26,v27,v28,v29,v30,v31 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -16755,39 +16436,39 @@ namespace simd { } // Saturated math - simdjson_really_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } - simdjson_really_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } // Order-specific operations - simdjson_really_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } - simdjson_really_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_really_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_really_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_really_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } // Bit-specific operations - simdjson_really_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_really_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } - simdjson_really_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } - simdjson_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } - simdjson_really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } template - simdjson_really_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + simdjson_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } template - simdjson_really_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + simdjson_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } // Get one of the bits and make a bitmask out of it. // e.g. value.get_bit<7>() gets the high bit template - simdjson_really_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } + simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); } }; template @@ -16800,10 +16481,10 @@ namespace simd { simd8x64& operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} - simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{chunk0, chunk1} {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+32)} {} - simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const { + simdjson_inline uint64_t compress(uint64_t mask, T * output) const { uint32_t mask1 = uint32_t(mask); uint32_t mask2 = uint32_t(mask >> 32); this->chunks[0].compress(mask1, output); @@ -16811,22 +16492,22 @@ namespace simd { return 64 - count_ones(mask); } - simdjson_really_inline void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); this->chunks[1].store(ptr+sizeof(simd8)*1); } - simdjson_really_inline uint64_t to_bitmask() const { + simdjson_inline uint64_t to_bitmask() const { uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); uint64_t r_hi = this->chunks[1].to_bitmask(); return r_lo | (r_hi << 32); } - simdjson_really_inline simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return this->chunks[0] | this->chunks[1]; } - simdjson_really_inline simd8x64 bit_or(const T m) const { + simdjson_inline simd8x64 bit_or(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] | mask, @@ -16834,7 +16515,7 @@ namespace simd { ); } - simdjson_really_inline uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] == mask, @@ -16842,14 +16523,14 @@ namespace simd { ).to_bitmask(); } - simdjson_really_inline uint64_t eq(const simd8x64 &other) const { + simdjson_inline uint64_t eq(const simd8x64 &other) const { return simd8x64( this->chunks[0] == other.chunks[0], this->chunks[1] == other.chunks[1] ).to_bitmask(); } - simdjson_really_inline uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] <= mask, @@ -16875,11 +16556,11 @@ namespace jsoncharutils { // return non-zero if not a structural or whitespace char // zero otherwise -simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace_negated[c]; } -simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace[c]; } @@ -16910,7 +16591,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -16942,10 +16623,10 @@ simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -16959,7 +16640,7 @@ static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64 using internal::value128; -simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -16995,13 +16676,13 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. simdjson_warn_unused -simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); @@ -17009,36 +16690,36 @@ simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -17064,18 +16745,18 @@ using namespace simd; struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_really_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } uint32_t bs_bits; uint32_t quote_bits; }; // struct backslash_and_quote -simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { +simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 15 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); @@ -17092,147 +16773,6 @@ simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(co } // namespace haswell } // namespace simdjson -/* begin file include/simdjson/generic/stringparsing.h */ -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace haswell { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // check for low surrogate for characters outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - - // if the first code point is invalid we will get here, as we will go past - // the check for being outside the Basic Multilingual plane. If we don't - // find a \u immediately afterwards we fail out anyhow, but if we do, - // this check catches both the case of the first code point being invalid - // or the second code point being invalid. - if ((code_point | code_point_2) >> 16) { - return false; - } - - code_point = - (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000; - *src_ptr += 6; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - -/** - * Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then - * dst needs to have four free bytes. - */ -simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - /* can't be reached */ - return nullptr; -} - -simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { - if (*(src++) != '"') { return STRING_ERROR; } - auto end = stringparsing::parse_string(src, current_string_buf_loc); - if (!end) { return STRING_ERROR; } - s = std::string_view(reinterpret_cast(current_string_buf_loc), end-current_string_buf_loc); - current_string_buf_loc = end; - return SUCCESS; -} - -} // namespace stringparsing -} // unnamed namespace -} // namespace haswell -} // namespace simdjson -/* end file include/simdjson/generic/stringparsing.h */ - #endif // SIMDJSON_HASWELL_STRINGPARSING_H /* end file include/simdjson/haswell/stringparsing.h */ /* begin file include/simdjson/haswell/numberparsing.h */ @@ -17243,7 +16783,7 @@ namespace simdjson { namespace haswell { namespace { -static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { +static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); const __m128i mul_1_10 = @@ -17306,7 +16846,7 @@ namespace { // Convert a mantissa, an exponent and a sign bit into an ieee64 double. // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { +simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { double d; mantissa &= ~(1ULL << 52); mantissa |= real_exponent << 52; @@ -17321,7 +16861,7 @@ simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponen // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [smallest_power, // largest_power] interval: the caller is responsible for this check. -simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -17381,7 +16921,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg // In the slow path, we need to adjust i so that it is > 1<<63 which is always // possible, except if i == 0, so we handle i == 0 separately. if(i == 0) { - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } @@ -17496,7 +17036,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? // Here have that real_exponent <= 0 so -real_exponent >= 0 if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } // next line is safe because -real_exponent + 1 < 0 @@ -17604,7 +17144,7 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -17631,7 +17171,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) { template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -17641,7 +17181,7 @@ simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -17669,7 +17209,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c return SUCCESS; } -simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -17720,7 +17260,7 @@ simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t * return SUCCESS; } -simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { +simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -17730,7 +17270,7 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s } template -simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { +simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. @@ -17766,7 +17306,8 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); // if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - WRITE_DOUBLE(0, src, writer); + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); return SUCCESS; } else { // (exponent > largest_power) and (i != 0) // We have, for sure, an infinite value and simdjson refuses to parse infinite values. @@ -17786,20 +17327,20 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg #ifdef SIMDJSON_SKIPNUMBERPARSING template -simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } #else // parse the number at src @@ -17812,13 +17353,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge // // Our objective is accurate parsing (ULP of 0) at high speed. template -simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -17962,7 +17503,7 @@ const uint8_t integer_string_finisher[256] = { NUMBER_ERROR}; // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -18012,7 +17553,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( // Parse any number from 0 to 18,446,744,073,709,551,615 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -18060,7 +17601,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( } // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { const uint8_t *p = src + 1; // // Parse the integer part. @@ -18110,12 +17651,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -18153,13 +17694,13 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { // // Check for minus sign // if(src == src_end) { return NUMBER_ERROR; } bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -18196,24 +17737,24 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - const uint8_t *p = src + negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. // // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; + const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + while (parse_digit(*src, i)) { src++; } // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); + size_t digit_count = size_t(src - start_digits); // We go from // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // so we can never represent numbers that have more than 19 digits. @@ -18225,11 +17766,11 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in // Here digit_count > 0. if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; // } // as a single table lookup: - if(*p != '"') { return NUMBER_ERROR; } + if(*src != '"') { return NUMBER_ERROR; } // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. // Performance note: This check is only needed when digit_count == longest_digit_count but it is // so cheap that we might as well always make it. @@ -18237,12 +17778,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in return negative ? (~i+1) : i; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -18309,19 +17850,19 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -18329,9 +17870,9 @@ simdjson_unused simdjson_really_inline simdjson_result is_integer(const ui return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -18355,13 +17896,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge } // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { if(src == src_end) { return NUMBER_ERROR; } // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -18430,18 +17971,18 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, src_end, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - src += negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. @@ -18508,7 +18049,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double_in_s if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; @@ -18561,7 +18102,7 @@ using namespace simdjson::dom; class implementation final : public simdjson::implementation { public: - simdjson_really_inline implementation() + simdjson_inline implementation() : simdjson::implementation("ppc64", "PPC64 ALTIVEC", internal::instruction_set::ALTIVEC) {} simdjson_warn_unused error_code create_dom_parser_implementation( @@ -18623,10 +18164,11 @@ class dom_parser_implementation final : public internal::dom_parser_implementati simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: - simdjson_really_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); }; @@ -18685,6 +18227,8 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth( #undef vector #endif +static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); + #endif // SIMDJSON_PPC64_INTRINSICS_H /* end file include/simdjson/ppc64/intrinsics.h */ /* begin file include/simdjson/ppc64/bitmanipulation.h */ @@ -18699,7 +18243,7 @@ namespace { // but the algorithms do not end up using the returned value. // Sadly, sanitizers are not smart enough to figure it out. SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_really_inline int trailing_zeroes(uint64_t input_num) { +simdjson_inline int trailing_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long ret; // Search the mask data from least significant bit (LSB) @@ -18712,12 +18256,12 @@ simdjson_really_inline int trailing_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { +simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { return input_num & (input_num - 1); } /* result might be undefined when input_num is zero */ -simdjson_really_inline int leading_zeroes(uint64_t input_num) { +simdjson_inline int leading_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long leading_zero = 0; // Search the mask data from most significant bit (MSB) @@ -18732,17 +18276,17 @@ simdjson_really_inline int leading_zeroes(uint64_t input_num) { } #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_really_inline int count_ones(uint64_t input_num) { +simdjson_inline int count_ones(uint64_t input_num) { // note: we do not support legacy 32-bit Windows return __popcnt64(input_num); // Visual Studio wants two underscores } #else -simdjson_really_inline int count_ones(uint64_t input_num) { +simdjson_inline int count_ones(uint64_t input_num) { return __builtin_popcountll(input_num); } #endif -simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, +simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO *result = value1 + value2; @@ -18773,7 +18317,7 @@ namespace { // // For example, prefix_xor(00100100) == 00011100 // -simdjson_really_inline uint64_t prefix_xor(uint64_t bitmask) { +simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { // You can use the version below, however gcc sometimes miscompiles // vec_pmsum_be, it happens somewhere around between 8 and 9th version. // The performance boost was not noticeable, falling back to a usual @@ -18820,41 +18364,41 @@ template struct base { __m128i value; // Zero constructor - simdjson_really_inline base() : value{__m128i()} {} + simdjson_inline base() : value{__m128i()} {} // Conversion from SIMD register - simdjson_really_inline base(const __m128i _value) : value(_value) {} + simdjson_inline base(const __m128i _value) : value(_value) {} // Conversion to SIMD register - simdjson_really_inline operator const __m128i &() const { + simdjson_inline operator const __m128i &() const { return this->value; } - simdjson_really_inline operator __m128i &() { return this->value; } + simdjson_inline operator __m128i &() { return this->value; } // Bit operations - simdjson_really_inline Child operator|(const Child other) const { + simdjson_inline Child operator|(const Child other) const { return vec_or(this->value, (__m128i)other); } - simdjson_really_inline Child operator&(const Child other) const { + simdjson_inline Child operator&(const Child other) const { return vec_and(this->value, (__m128i)other); } - simdjson_really_inline Child operator^(const Child other) const { + simdjson_inline Child operator^(const Child other) const { return vec_xor(this->value, (__m128i)other); } - simdjson_really_inline Child bit_andnot(const Child other) const { + simdjson_inline Child bit_andnot(const Child other) const { return vec_andc(this->value, (__m128i)other); } - simdjson_really_inline Child &operator|=(const Child other) { + simdjson_inline Child &operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_really_inline Child &operator&=(const Child other) { + simdjson_inline Child &operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_really_inline Child &operator^=(const Child other) { + simdjson_inline Child &operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; @@ -18869,17 +18413,17 @@ struct base8 : base> { typedef uint16_t bitmask_t; typedef uint32_t bitmask2_t; - simdjson_really_inline base8() : base>() {} - simdjson_really_inline base8(const __m128i _value) : base>(_value) {} + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs); } static const int SIZE = sizeof(base>::value); template - simdjson_really_inline simd8 prev(simd8 prev_chunk) const { + simdjson_inline simd8 prev(simd8 prev_chunk) const { __m128i chunk = this->value; #ifdef __LITTLE_ENDIAN__ chunk = (__m128i)vec_reve(this->value); @@ -18895,18 +18439,18 @@ struct base8 : base> { // SIMD byte mask type (returned by things like eq and gt) template <> struct simd8 : base8 { - static simdjson_really_inline simd8 splat(bool _value) { + static simdjson_inline simd8 splat(bool _value) { return (__m128i)vec_splats((unsigned char)(-(!!_value))); } - simdjson_really_inline simd8() : base8() {} - simdjson_really_inline simd8(const __m128i _value) + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} // Splat constructor - simdjson_really_inline simd8(bool _value) + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_really_inline int to_bitmask() const { + simdjson_inline int to_bitmask() const { __vector unsigned long long result; const __m128i perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00}; @@ -18919,25 +18463,25 @@ template <> struct simd8 : base8 { return static_cast(result[0]); #endif } - simdjson_really_inline bool any() const { + simdjson_inline bool any() const { return !vec_all_eq(this->value, (__m128i)vec_splats(0)); } - simdjson_really_inline simd8 operator~() const { + simdjson_inline simd8 operator~() const { return this->value ^ (__m128i)splat(true); } }; template struct base8_numeric : base8 { - static simdjson_really_inline simd8 splat(T value) { + static simdjson_inline simd8 splat(T value) { (void)value; return (__m128i)vec_splats(value); } - static simdjson_really_inline simd8 zero() { return splat(0); } - static simdjson_really_inline simd8 load(const T values[16]) { + static simdjson_inline simd8 zero() { return splat(0); } + static simdjson_inline simd8 load(const T values[16]) { return (__m128i)(vec_vsx_ld(0, reinterpret_cast(values))); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_really_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, + static simdjson_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15) { @@ -18945,30 +18489,30 @@ template struct base8_numeric : base8 { v14, v15); } - simdjson_really_inline base8_numeric() : base8() {} - simdjson_really_inline base8_numeric(const __m128i _value) + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} // Store to array - simdjson_really_inline void store(T dst[16]) const { + simdjson_inline void store(T dst[16]) const { vec_vsx_st(this->value, 0, reinterpret_cast<__m128i *>(dst)); } // Override to distinguish from bool version - simdjson_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } // Addition/subtraction are the same for signed and unsigned - simdjson_really_inline simd8 operator+(const simd8 other) const { + simdjson_inline simd8 operator+(const simd8 other) const { return (__m128i)((__m128i)this->value + (__m128i)other); } - simdjson_really_inline simd8 operator-(const simd8 other) const { + simdjson_inline simd8 operator-(const simd8 other) const { return (__m128i)((__m128i)this->value - (__m128i)other); } - simdjson_really_inline simd8 &operator+=(const simd8 other) { + simdjson_inline simd8 &operator+=(const simd8 other) { *this = *this + other; return *static_cast *>(this); } - simdjson_really_inline simd8 &operator-=(const simd8 other) { + simdjson_inline simd8 &operator-=(const simd8 other) { *this = *this - other; return *static_cast *>(this); } @@ -18976,7 +18520,7 @@ template struct base8_numeric : base8 { // Perform a lookup assuming the value is between 0 and 16 (undefined behavior // for out of range values) template - simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { return (__m128i)vec_perm((__m128i)lookup_table, (__m128i)lookup_table, this->value); } @@ -18987,7 +18531,7 @@ template struct base8_numeric : base8 { // seems like a function with the signature simd8 compress(uint32_t mask) // would be sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_really_inline void compress(uint16_t mask, L *output) const { + simdjson_inline void compress(uint16_t mask, L *output) const { using internal::BitsSetTable256mul2; using internal::pshufb_combine_table; using internal::thintable_epi8; @@ -19026,7 +18570,7 @@ template struct base8_numeric : base8 { } template - simdjson_really_inline simd8 + simdjson_inline simd8 lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, L replace12, L replace13, L replace14, @@ -19040,15 +18584,15 @@ template struct base8_numeric : base8 { // Signed bytes template <> struct simd8 : base8_numeric { - simdjson_really_inline simd8() : base8_numeric() {} - simdjson_really_inline simd8(const __m128i _value) + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} // Splat constructor - simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_really_inline simd8(const int8_t *values) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t *values) : simd8(load(values)) {} // Member-by-member initialization - simdjson_really_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, + simdjson_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15) @@ -19056,7 +18600,7 @@ template <> struct simd8 : base8_numeric { v8, v9, v10, v11, v12, v13, v14, v15}) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 + simdjson_inline static simd8 repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15) { @@ -19065,22 +18609,22 @@ template <> struct simd8 : base8_numeric { } // Order-sensitive comparisons - simdjson_really_inline simd8 + simdjson_inline simd8 max_val(const simd8 other) const { return (__m128i)vec_max((__vector signed char)this->value, (__vector signed char)(__m128i)other); } - simdjson_really_inline simd8 + simdjson_inline simd8 min_val(const simd8 other) const { return (__m128i)vec_min((__vector signed char)this->value, (__vector signed char)(__m128i)other); } - simdjson_really_inline simd8 + simdjson_inline simd8 operator>(const simd8 other) const { return (__m128i)vec_cmpgt((__vector signed char)this->value, (__vector signed char)(__m128i)other); } - simdjson_really_inline simd8 + simdjson_inline simd8 operator<(const simd8 other) const { return (__m128i)vec_cmplt((__vector signed char)this->value, (__vector signed char)(__m128i)other); @@ -19089,22 +18633,22 @@ template <> struct simd8 : base8_numeric { // Unsigned bytes template <> struct simd8 : base8_numeric { - simdjson_really_inline simd8() : base8_numeric() {} - simdjson_really_inline simd8(const __m128i _value) + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} // Splat constructor - simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_really_inline simd8(const uint8_t *values) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t *values) : simd8(load(values)) {} // Member-by-member initialization - simdjson_really_inline + simdjson_inline simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) : simd8((__m128i){v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15}) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 + simdjson_inline static simd8 repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, @@ -19114,82 +18658,82 @@ template <> struct simd8 : base8_numeric { } // Saturated math - simdjson_really_inline simd8 + simdjson_inline simd8 saturating_add(const simd8 other) const { return (__m128i)vec_adds(this->value, (__m128i)other); } - simdjson_really_inline simd8 + simdjson_inline simd8 saturating_sub(const simd8 other) const { return (__m128i)vec_subs(this->value, (__m128i)other); } // Order-specific operations - simdjson_really_inline simd8 + simdjson_inline simd8 max_val(const simd8 other) const { return (__m128i)vec_max(this->value, (__m128i)other); } - simdjson_really_inline simd8 + simdjson_inline simd8 min_val(const simd8 other) const { return (__m128i)vec_min(this->value, (__m128i)other); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_really_inline simd8 + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_really_inline simd8 + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_really_inline simd8 + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_really_inline simd8 + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_really_inline simd8 + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_really_inline simd8 + simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } // Bit-specific operations - simdjson_really_inline simd8 bits_not_set() const { + simdjson_inline simd8 bits_not_set() const { return (__m128i)vec_cmpeq(this->value, (__m128i)vec_splats(uint8_t(0))); } - simdjson_really_inline simd8 bits_not_set(simd8 bits) const { + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_really_inline simd8 any_bits_set() const { + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_really_inline simd8 any_bits_set(simd8 bits) const { + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_really_inline bool bits_not_set_anywhere() const { + simdjson_inline bool bits_not_set_anywhere() const { return vec_all_eq(this->value, (__m128i)vec_splats(0)); } - simdjson_really_inline bool any_bits_set_anywhere() const { + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_really_inline bool bits_not_set_anywhere(simd8 bits) const { + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return vec_all_eq(vec_and(this->value, (__m128i)bits), (__m128i)vec_splats(0)); } - simdjson_really_inline bool any_bits_set_anywhere(simd8 bits) const { + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template simdjson_really_inline simd8 shr() const { + template simdjson_inline simd8 shr() const { return simd8( (__m128i)vec_sr(this->value, (__m128i)vec_splat_u8(N))); } - template simdjson_really_inline simd8 shl() const { + template simdjson_inline simd8 shl() const { return simd8( (__m128i)vec_sl(this->value, (__m128i)vec_splat_u8(N))); } @@ -19206,26 +18750,26 @@ template struct simd8x64 { operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_really_inline simd8x64(const T ptr[64]) + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48)} {} - simdjson_really_inline void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr + sizeof(simd8) * 0); this->chunks[1].store(ptr + sizeof(simd8) * 1); this->chunks[2].store(ptr + sizeof(simd8) * 2); this->chunks[3].store(ptr + sizeof(simd8) * 3); } - simdjson_really_inline simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); } - simdjson_really_inline uint64_t compress(uint64_t mask, T *output) const { + simdjson_inline uint64_t compress(uint64_t mask, T *output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); @@ -19236,7 +18780,7 @@ template struct simd8x64 { return 64 - count_ones(mask); } - simdjson_really_inline uint64_t to_bitmask() const { + simdjson_inline uint64_t to_bitmask() const { uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); uint64_t r1 = this->chunks[1].to_bitmask(); uint64_t r2 = this->chunks[2].to_bitmask(); @@ -19244,14 +18788,14 @@ template struct simd8x64 { return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); } - simdjson_really_inline uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, this->chunks[2] == mask, this->chunks[3] == mask) .to_bitmask(); } - simdjson_really_inline uint64_t eq(const simd8x64 &other) const { + simdjson_inline uint64_t eq(const simd8x64 &other) const { return simd8x64(this->chunks[0] == other.chunks[0], this->chunks[1] == other.chunks[1], this->chunks[2] == other.chunks[2], @@ -19259,7 +18803,7 @@ template struct simd8x64 { .to_bitmask(); } - simdjson_really_inline uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, this->chunks[2] <= mask, this->chunks[3] <= mask) @@ -19283,11 +18827,11 @@ namespace jsoncharutils { // return non-zero if not a structural or whitespace char // zero otherwise -simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace_negated[c]; } -simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace[c]; } @@ -19318,7 +18862,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -19350,10 +18894,10 @@ simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -19367,7 +18911,7 @@ static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64 using internal::value128; -simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -19403,13 +18947,13 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. simdjson_warn_unused -simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); @@ -19417,36 +18961,36 @@ simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -19472,17 +19016,17 @@ using namespace simd; struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_really_inline static backslash_and_quote + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - simdjson_really_inline bool has_quote_first() { + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_really_inline bool has_backslash() { return bs_bits != 0; } - simdjson_really_inline int quote_index() { + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_really_inline int backslash_index() { + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } @@ -19490,7 +19034,7 @@ struct backslash_and_quote { uint32_t quote_bits; }; // struct backslash_and_quote -simdjson_really_inline backslash_and_quote +simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -19517,147 +19061,6 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { } // namespace ppc64 } // namespace simdjson -/* begin file include/simdjson/generic/stringparsing.h */ -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace ppc64 { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // check for low surrogate for characters outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - - // if the first code point is invalid we will get here, as we will go past - // the check for being outside the Basic Multilingual plane. If we don't - // find a \u immediately afterwards we fail out anyhow, but if we do, - // this check catches both the case of the first code point being invalid - // or the second code point being invalid. - if ((code_point | code_point_2) >> 16) { - return false; - } - - code_point = - (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000; - *src_ptr += 6; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - -/** - * Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then - * dst needs to have four free bytes. - */ -simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - /* can't be reached */ - return nullptr; -} - -simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { - if (*(src++) != '"') { return STRING_ERROR; } - auto end = stringparsing::parse_string(src, current_string_buf_loc); - if (!end) { return STRING_ERROR; } - s = std::string_view(reinterpret_cast(current_string_buf_loc), end-current_string_buf_loc); - current_string_buf_loc = end; - return SUCCESS; -} - -} // namespace stringparsing -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson -/* end file include/simdjson/generic/stringparsing.h */ - #endif // SIMDJSON_PPC64_STRINGPARSING_H /* end file include/simdjson/ppc64/stringparsing.h */ /* begin file include/simdjson/ppc64/numberparsing.h */ @@ -19676,7 +19079,7 @@ namespace { // we don't have appropriate instructions, so let us use a scalar function // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ -static simdjson_really_inline uint32_t +static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { uint64_t val; std::memcpy(&val, chars, sizeof(uint64_t)); @@ -19737,7 +19140,7 @@ namespace { // Convert a mantissa, an exponent and a sign bit into an ieee64 double. // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { +simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { double d; mantissa &= ~(1ULL << 52); mantissa |= real_exponent << 52; @@ -19752,7 +19155,7 @@ simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponen // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [smallest_power, // largest_power] interval: the caller is responsible for this check. -simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -19812,7 +19215,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg // In the slow path, we need to adjust i so that it is > 1<<63 which is always // possible, except if i == 0, so we handle i == 0 separately. if(i == 0) { - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } @@ -19927,7 +19330,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? // Here have that real_exponent <= 0 so -real_exponent >= 0 if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } // next line is safe because -real_exponent + 1 < 0 @@ -20035,7 +19438,7 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -20062,7 +19465,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) { template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -20072,7 +19475,7 @@ simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -20100,7 +19503,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c return SUCCESS; } -simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -20151,7 +19554,7 @@ simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t * return SUCCESS; } -simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { +simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -20161,7 +19564,7 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s } template -simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { +simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. @@ -20197,7 +19600,8 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); // if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - WRITE_DOUBLE(0, src, writer); + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); return SUCCESS; } else { // (exponent > largest_power) and (i != 0) // We have, for sure, an infinite value and simdjson refuses to parse infinite values. @@ -20217,20 +19621,20 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg #ifdef SIMDJSON_SKIPNUMBERPARSING template -simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } #else // parse the number at src @@ -20243,13 +19647,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge // // Our objective is accurate parsing (ULP of 0) at high speed. template -simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -20393,7 +19797,7 @@ const uint8_t integer_string_finisher[256] = { NUMBER_ERROR}; // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -20443,7 +19847,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( // Parse any number from 0 to 18,446,744,073,709,551,615 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -20491,7 +19895,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( } // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { const uint8_t *p = src + 1; // // Parse the integer part. @@ -20541,12 +19945,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -20584,13 +19988,13 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { // // Check for minus sign // if(src == src_end) { return NUMBER_ERROR; } bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -20627,24 +20031,24 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - const uint8_t *p = src + negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. // // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; + const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + while (parse_digit(*src, i)) { src++; } // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); + size_t digit_count = size_t(src - start_digits); // We go from // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // so we can never represent numbers that have more than 19 digits. @@ -20656,11 +20060,11 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in // Here digit_count > 0. if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; // } // as a single table lookup: - if(*p != '"') { return NUMBER_ERROR; } + if(*src != '"') { return NUMBER_ERROR; } // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. // Performance note: This check is only needed when digit_count == longest_digit_count but it is // so cheap that we might as well always make it. @@ -20668,12 +20072,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in return negative ? (~i+1) : i; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -20740,19 +20144,19 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -20760,9 +20164,9 @@ simdjson_unused simdjson_really_inline simdjson_result is_integer(const ui return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -20786,13 +20190,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge } // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { if(src == src_end) { return NUMBER_ERROR; } // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -20861,18 +20265,18 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, src_end, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - src += negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. @@ -20939,7 +20343,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double_in_s if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; @@ -21004,7 +20408,7 @@ using namespace simdjson::dom; class implementation final : public simdjson::implementation { public: - simdjson_really_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42 | internal::instruction_set::PCLMULQDQ) {} + simdjson_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42 | internal::instruction_set::PCLMULQDQ) {} simdjson_warn_unused error_code create_dom_parser_implementation( size_t capacity, size_t max_length, @@ -21044,7 +20448,7 @@ class implementation final : public simdjson::implementation { #include // for _mm_clmulepi64_si128 #endif - +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); #endif // SIMDJSON_WESTMERE_INTRINSICS_H /* end file include/simdjson/westmere/intrinsics.h */ @@ -21095,10 +20499,11 @@ class dom_parser_implementation final : public internal::dom_parser_implementati simdjson_warn_unused error_code stage1(const uint8_t *buf, size_t len, stage1_mode partial) noexcept final; simdjson_warn_unused error_code stage2(dom::document &doc) noexcept final; simdjson_warn_unused error_code stage2_next(dom::document &doc) noexcept final; + simdjson_warn_unused uint8_t *parse_string(const uint8_t *src, uint8_t *dst) const noexcept final; inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; private: - simdjson_really_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); }; @@ -21151,7 +20556,7 @@ namespace { // but the algorithms do not end up using the returned value. // Sadly, sanitizers are not smart enough to figure it out. SIMDJSON_NO_SANITIZE_UNDEFINED -simdjson_really_inline int trailing_zeroes(uint64_t input_num) { +simdjson_inline int trailing_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long ret; // Search the mask data from least significant bit (LSB) @@ -21164,12 +20569,12 @@ simdjson_really_inline int trailing_zeroes(uint64_t input_num) { } /* result might be undefined when input_num is zero */ -simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) { +simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { return input_num & (input_num-1); } /* result might be undefined when input_num is zero */ -simdjson_really_inline int leading_zeroes(uint64_t input_num) { +simdjson_inline int leading_zeroes(uint64_t input_num) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO unsigned long leading_zero = 0; // Search the mask data from most significant bit (MSB) @@ -21184,17 +20589,17 @@ simdjson_really_inline int leading_zeroes(uint64_t input_num) { } #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO -simdjson_really_inline unsigned __int64 count_ones(uint64_t input_num) { +simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { // note: we do not support legacy 32-bit Windows return __popcnt64(input_num);// Visual Studio wants two underscores } #else -simdjson_really_inline long long int count_ones(uint64_t input_num) { +simdjson_inline long long int count_ones(uint64_t input_num) { return _popcnt64(input_num); } #endif -simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, +simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO return _addcarry_u64(0, value1, value2, @@ -21224,7 +20629,7 @@ namespace { // // For example, prefix_xor(00100100) == 00011100 // -simdjson_really_inline uint64_t prefix_xor(const uint64_t bitmask) { +simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { // There should be no such thing with a processing supporting avx2 // but not clmul. __m128i all_ones = _mm_set1_epi8('\xFF'); @@ -21253,23 +20658,23 @@ namespace simd { __m128i value; // Zero constructor - simdjson_really_inline base() : value{__m128i()} {} + simdjson_inline base() : value{__m128i()} {} // Conversion from SIMD register - simdjson_really_inline base(const __m128i _value) : value(_value) {} + simdjson_inline base(const __m128i _value) : value(_value) {} // Conversion to SIMD register - simdjson_really_inline operator const __m128i&() const { return this->value; } - simdjson_really_inline operator __m128i&() { return this->value; } + simdjson_inline operator const __m128i&() const { return this->value; } + simdjson_inline operator __m128i&() { return this->value; } // Bit operations - simdjson_really_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_really_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_really_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_really_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_really_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_really_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_really_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } }; // Forward-declared so they can be used by splat and friends. @@ -21281,15 +20686,15 @@ namespace simd { typedef uint16_t bitmask_t; typedef uint32_t bitmask2_t; - simdjson_really_inline base8() : base>() {} - simdjson_really_inline base8(const __m128i _value) : base>(_value) {} + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } static const int SIZE = sizeof(base>::value); template - simdjson_really_inline simd8 prev(const simd8 prev_chunk) const { + simdjson_inline simd8 prev(const simd8 prev_chunk) const { return _mm_alignr_epi8(*this, prev_chunk, 16 - N); } }; @@ -21297,27 +20702,27 @@ namespace simd { // SIMD byte mask type (returned by things like eq and gt) template<> struct simd8: base8 { - static simdjson_really_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } + static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - simdjson_really_inline simd8() : base8() {} - simdjson_really_inline simd8(const __m128i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} // Splat constructor - simdjson_really_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_really_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_really_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_really_inline simd8 operator~() const { return *this ^ true; } + simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } }; template struct base8_numeric: base8 { - static simdjson_really_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_really_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_really_inline simd8 load(const T values[16]) { + static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } + static simdjson_inline simd8 load(const T values[16]) { return _mm_loadu_si128(reinterpret_cast(values)); } // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_really_inline simd8 repeat_16( + static simdjson_inline simd8 repeat_16( T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 ) { @@ -21327,24 +20732,24 @@ namespace simd { ); } - simdjson_really_inline base8_numeric() : base8() {} - simdjson_really_inline base8_numeric(const __m128i _value) : base8(_value) {} + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} // Store to array - simdjson_really_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } + simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); } // Override to distinguish from bool version - simdjson_really_inline simd8 operator~() const { return *this ^ 0xFFu; } + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } // Addition/subtraction are the same for signed and unsigned - simdjson_really_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_really_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_really_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_really_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) template - simdjson_really_inline simd8 lookup_16(simd8 lookup_table) const { + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { return _mm_shuffle_epi8(lookup_table, *this); } @@ -21356,7 +20761,7 @@ namespace simd { // signature simd8 compress(uint32_t mask) would be // sensible, but the AVX ISA makes this kind of approach difficult. template - simdjson_really_inline void compress(uint16_t mask, L * output) const { + simdjson_inline void compress(uint16_t mask, L * output) const { using internal::thintable_epi8; using internal::BitsSetTable256mul2; using internal::pshufb_combine_table; @@ -21387,7 +20792,7 @@ namespace simd { } template - simdjson_really_inline simd8 lookup_16( + simdjson_inline simd8 lookup_16( L replace0, L replace1, L replace2, L replace3, L replace4, L replace5, L replace6, L replace7, L replace8, L replace9, L replace10, L replace11, @@ -21404,14 +20809,14 @@ namespace simd { // Signed bytes template<> struct simd8 : base8_numeric { - simdjson_really_inline simd8() : base8_numeric() {} - simdjson_really_inline simd8(const __m128i _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} // Splat constructor - simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_really_inline simd8(const int8_t* values) : simd8(load(values)) {} + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} // Member-by-member initialization - simdjson_really_inline simd8( + simdjson_inline simd8( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) : simd8(_mm_setr_epi8( @@ -21419,7 +20824,7 @@ namespace simd { v8, v9, v10,v11,v12,v13,v14,v15 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 ) { @@ -21430,23 +20835,23 @@ namespace simd { } // Order-sensitive comparisons - simdjson_really_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_really_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_really_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_really_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } }; // Unsigned bytes template<> struct simd8: base8_numeric { - simdjson_really_inline simd8() : base8_numeric() {} - simdjson_really_inline simd8(const __m128i _value) : base8_numeric(_value) {} + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} // Splat constructor - simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} // Array constructor - simdjson_really_inline simd8(const uint8_t* values) : simd8(load(values)) {} + simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} // Member-by-member initialization - simdjson_really_inline simd8( + simdjson_inline simd8( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) : simd8(_mm_setr_epi8( @@ -21454,7 +20859,7 @@ namespace simd { v8, v9, v10,v11,v12,v13,v14,v15 )) {} // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_really_inline static simd8 repeat_16( + simdjson_inline static simd8 repeat_16( uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 ) { @@ -21465,39 +20870,39 @@ namespace simd { } // Saturated math - simdjson_really_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_really_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } // Order-specific operations - simdjson_really_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_really_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_really_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_really_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_really_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_really_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_really_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_really_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } // Bit-specific operations - simdjson_really_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_really_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_really_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_really_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_really_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_really_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_really_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_really_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } template - simdjson_really_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } template - simdjson_really_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } // Get one of the bits and make a bitmask out of it. // e.g. value.get_bit<7>() gets the high bit template - simdjson_really_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } + simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); } }; template @@ -21510,21 +20915,21 @@ namespace simd { simd8x64& operator=(const simd8& other) = delete; // no assignment allowed simd8x64() = delete; // no default constructor allowed - simdjson_really_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} - simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{simd8::load(ptr), simd8::load(ptr+16), simd8::load(ptr+32), simd8::load(ptr+48)} {} - simdjson_really_inline void store(T ptr[64]) const { + simdjson_inline void store(T ptr[64]) const { this->chunks[0].store(ptr+sizeof(simd8)*0); this->chunks[1].store(ptr+sizeof(simd8)*1); this->chunks[2].store(ptr+sizeof(simd8)*2); this->chunks[3].store(ptr+sizeof(simd8)*3); } - simdjson_really_inline simd8 reduce_or() const { + simdjson_inline simd8 reduce_or() const { return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); } - simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const { + simdjson_inline uint64_t compress(uint64_t mask, T * output) const { this->chunks[0].compress(uint16_t(mask), output); this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); @@ -21532,7 +20937,7 @@ namespace simd { return 64 - count_ones(mask); } - simdjson_really_inline uint64_t to_bitmask() const { + simdjson_inline uint64_t to_bitmask() const { uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() ); uint64_t r1 = this->chunks[1].to_bitmask() ; uint64_t r2 = this->chunks[2].to_bitmask() ; @@ -21540,7 +20945,7 @@ namespace simd { return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); } - simdjson_really_inline uint64_t eq(const T m) const { + simdjson_inline uint64_t eq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] == mask, @@ -21550,7 +20955,7 @@ namespace simd { ).to_bitmask(); } - simdjson_really_inline uint64_t eq(const simd8x64 &other) const { + simdjson_inline uint64_t eq(const simd8x64 &other) const { return simd8x64( this->chunks[0] == other.chunks[0], this->chunks[1] == other.chunks[1], @@ -21559,7 +20964,7 @@ namespace simd { ).to_bitmask(); } - simdjson_really_inline uint64_t lteq(const T m) const { + simdjson_inline uint64_t lteq(const T m) const { const simd8 mask = simd8::splat(m); return simd8x64( this->chunks[0] <= mask, @@ -21586,11 +20991,11 @@ namespace jsoncharutils { // return non-zero if not a structural or whitespace char // zero otherwise -simdjson_really_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace_negated[c]; } -simdjson_really_inline uint32_t is_structural_or_whitespace(uint8_t c) { +simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { return internal::structural_or_whitespace[c]; } @@ -21621,7 +21026,7 @@ static inline uint32_t hex_to_u32_nocheck( // // Note: we assume that surrogates are treated separately // -simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { +simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { if (cp <= 0x7F) { c[0] = uint8_t(cp); return 1; // ascii @@ -21653,10 +21058,10 @@ simdjson_really_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t *c) { #ifdef SIMDJSON_IS_32BITS // _umul128 for x86, arm // this is a slow emulation routine for 32-bit // -static simdjson_really_inline uint64_t __emulu(uint32_t x, uint32_t y) { +static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { return x * (uint64_t)y; } -static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { +static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); @@ -21670,7 +21075,7 @@ static simdjson_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64 using internal::value128; -simdjson_really_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { +simdjson_inline value128 full_multiplication(uint64_t value1, uint64_t value2) { value128 answer; #if defined(SIMDJSON_REGULAR_VISUAL_STUDIO) || defined(SIMDJSON_IS_32BITS) #ifdef _M_ARM64 @@ -21706,13 +21111,13 @@ namespace atomparsing { // You might think that using memcpy makes this function expensive, but you'd be wrong. // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); // to the compile-time constant 1936482662. -simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. simdjson_warn_unused -simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { +simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); std::memcpy(&srcval, src, sizeof(uint32_t)); @@ -21720,36 +21125,36 @@ simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_true_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_true_atom(src); } else if (len == 4) { return !str4ncmp(src, "true"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src) { return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_false_atom(const uint8_t *src, size_t len) { if (len > 5) { return is_valid_false_atom(src); } else if (len == 5) { return !str4ncmp(src+1, "alse"); } else { return false; } } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src) { return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; } simdjson_warn_unused -simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { +simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { if (len > 4) { return is_valid_null_atom(src); } else if (len == 4) { return !str4ncmp(src, "null"); } else { return false; } @@ -21774,18 +21179,18 @@ using namespace simd; struct backslash_and_quote { public: static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst); - simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_really_inline bool has_backslash() { return bs_bits != 0; } - simdjson_really_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_really_inline int backslash_index() { return trailing_zeroes(bs_bits); } + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } uint32_t bs_bits; uint32_t quote_bits; }; // struct backslash_and_quote -simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { +simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { // this can read up to 31 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); @@ -21804,147 +21209,6 @@ simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(co } // namespace westmere } // namespace simdjson -/* begin file include/simdjson/generic/stringparsing.h */ -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { -namespace westmere { -namespace { -/// @private -namespace stringparsing { - -// begin copypasta -// These chars yield themselves: " \ / -// b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab -// u not handled in this table as it's complex -static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -// handle a unicode codepoint -// write appropriate values into dest -// src will advance 6 bytes or 12 bytes -// dest will advance a variable amount (return via pointer) -// return true if the unicode codepoint was valid -// We work in little-endian then swap at write time -simdjson_warn_unused -simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr, - uint8_t **dst_ptr) { - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // check for low surrogate for characters outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') { - return false; - } - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - - // if the first code point is invalid we will get here, as we will go past - // the check for being outside the Basic Multilingual plane. If we don't - // find a \u immediately afterwards we fail out anyhow, but if we do, - // this check catches both the case of the first code point being invalid - // or the second code point being invalid. - if ((code_point | code_point_2) >> 16) { - return false; - } - - code_point = - (((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000; - *src_ptr += 6; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; -} - -/** - * Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then - * dst needs to have four free bytes. - */ -simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst)) { - return nullptr; - } - } else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - /* can't be reached */ - return nullptr; -} - -simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) { - if (*(src++) != '"') { return STRING_ERROR; } - auto end = stringparsing::parse_string(src, current_string_buf_loc); - if (!end) { return STRING_ERROR; } - s = std::string_view(reinterpret_cast(current_string_buf_loc), end-current_string_buf_loc); - current_string_buf_loc = end; - return SUCCESS; -} - -} // namespace stringparsing -} // unnamed namespace -} // namespace westmere -} // namespace simdjson -/* end file include/simdjson/generic/stringparsing.h */ - #endif // SIMDJSON_WESTMERE_STRINGPARSING_H /* end file include/simdjson/westmere/stringparsing.h */ /* begin file include/simdjson/westmere/numberparsing.h */ @@ -21955,7 +21219,7 @@ namespace simdjson { namespace westmere { namespace { -static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { +static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) { // this actually computes *16* values so we are being wasteful. const __m128i ascii0 = _mm_set1_epi8('0'); const __m128i mul_1_10 = @@ -22018,7 +21282,7 @@ namespace { // Convert a mantissa, an exponent and a sign bit into an ieee64 double. // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. -simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { +simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { double d; mantissa &= ~(1ULL << 52); mantissa |= real_exponent << 52; @@ -22033,7 +21297,7 @@ simdjson_really_inline double to_double(uint64_t mantissa, uint64_t real_exponen // set to false. This should work *most of the time* (like 99% of the time). // We assume that power is in the [smallest_power, // largest_power] interval: the caller is responsible for this check. -simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { +simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double &d) { // we start with a fast path // It was described in // Clinger WD. How to read floating point numbers accurately. @@ -22093,7 +21357,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg // In the slow path, we need to adjust i so that it is > 1<<63 which is always // possible, except if i == 0, so we handle i == 0 separately. if(i == 0) { - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } @@ -22208,7 +21472,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? // Here have that real_exponent <= 0 so -real_exponent >= 0 if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = 0.0; + d = negative ? -0.0 : 0.0; return true; } // next line is safe because -real_exponent + 1 < 0 @@ -22316,7 +21580,7 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html -simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { +simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { uint64_t val; // this can read up to 7 bytes beyond the buffer size, but we require // SIMDJSON_PADDING of padding @@ -22343,7 +21607,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) { template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later -simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { +simdjson_inline bool parse_digit(const uint8_t c, I &i) { const uint8_t digit = static_cast(c - '0'); if (digit > 9) { return false; @@ -22353,7 +21617,7 @@ simdjson_really_inline bool parse_digit(const uint8_t c, I &i) { return true; } -simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { +simdjson_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer // z that fits in 53 bits, then we will be able to convert back the @@ -22381,7 +21645,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c return SUCCESS; } -simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { +simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) { // Exp Sign: -123.456e[-]78 bool neg_exp = ('-' == *p); if (neg_exp || '+' == *p) { p++; } // Skip + as well @@ -22432,7 +21696,7 @@ simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t * return SUCCESS; } -simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { +simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; @@ -22442,7 +21706,7 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s } template -simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { +simdjson_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) { // If we frequently had to deal with long strings of digits, // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. @@ -22478,7 +21742,8 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); // if((exponent < simdjson::internal::smallest_power) || (i == 0)) { - WRITE_DOUBLE(0, src, writer); + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); return SUCCESS; } else { // (exponent > largest_power) and (i != 0) // We have, for sure, an infinite value and simdjson refuses to parse infinite values. @@ -22498,20 +21763,20 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg #ifdef SIMDJSON_SKIPNUMBERPARSING template -simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const, W &writer) { writer.append_s64(0); // always write zero return SUCCESS; // always succeeds } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return ondemand::number_type::signed_integer; } #else // parse the number at src @@ -22524,13 +21789,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge // // Our objective is accurate parsing (ULP of 0) at high speed. template -simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) { +simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -22674,7 +21939,7 @@ const uint8_t integer_string_finisher[256] = { NUMBER_ERROR}; // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -22724,7 +21989,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( // Parse any number from 0 to 18,446,744,073,709,551,615 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept { const uint8_t *p = src; // // Parse the integer part. @@ -22772,7 +22037,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned( } // Parse any number from 0 to 18,446,744,073,709,551,615 -simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { const uint8_t *p = src + 1; // // Parse the integer part. @@ -22822,12 +22087,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_unsigned_ } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -22865,13 +22130,13 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept { // // Check for minus sign // if(src == src_end) { return NUMBER_ERROR; } bool negative = (*src == '-'); - const uint8_t *p = src + negative; + const uint8_t *p = src + uint8_t(negative); // // Parse the integer part. @@ -22908,24 +22173,24 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer(co } // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 -simdjson_unused simdjson_really_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t *src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - const uint8_t *p = src + negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. // // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t *const start_digits = p; + const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + while (parse_digit(*src, i)) { src++; } // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); + size_t digit_count = size_t(src - start_digits); // We go from // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 // so we can never represent numbers that have more than 19 digits. @@ -22937,11 +22202,11 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in // Here digit_count > 0. if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; // } // as a single table lookup: - if(*p != '"') { return NUMBER_ERROR; } + if(*src != '"') { return NUMBER_ERROR; } // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. // Performance note: This check is only needed when digit_count == longest_digit_count but it is // so cheap that we might as well always make it. @@ -22949,12 +22214,12 @@ simdjson_unused simdjson_really_inline simdjson_result parse_integer_in return negative ? (~i+1) : i; } -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -23021,19 +22286,19 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline bool is_negative(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } -simdjson_unused simdjson_really_inline simdjson_result is_integer(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -23041,9 +22306,9 @@ simdjson_unused simdjson_really_inline simdjson_result is_integer(const ui return false; } -simdjson_unused simdjson_really_inline simdjson_result get_number_type(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); const uint8_t *p = src; while(static_cast(*p - '0') <= 9) { p++; } if ( p == src ) { return NUMBER_ERROR; } @@ -23067,13 +22332,13 @@ simdjson_unused simdjson_really_inline simdjson_result ge } // Never read at src_end or beyond -simdjson_unused simdjson_really_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { if(src == src_end) { return NUMBER_ERROR; } // // Check for minus sign // bool negative = (*src == '-'); - src += negative; + src += uint8_t(negative); // // Parse the integer part. @@ -23142,18 +22407,18 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double(cons if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, src_end, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { return NUMBER_ERROR; } return d; } -simdjson_unused simdjson_really_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { // // Check for minus sign // bool negative = (*(src + 1) == '-'); - src += negative + 1; + src += uint8_t(negative) + 1; // // Parse the integer part. @@ -23220,7 +22485,7 @@ simdjson_unused simdjson_really_inline simdjson_result parse_double_in_s if (simdjson_likely(!overflow)) { if (compute_float_64(exponent, i, negative, d)) { return d; } } - if (!parse_float_fallback(src-negative, &d)) { + if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; } return d; @@ -23309,22 +22574,22 @@ struct implementation_simdjson_result_base { /** * Create a new empty result with error = UNINITIALIZED. */ - simdjson_really_inline implementation_simdjson_result_base() noexcept = default; + simdjson_inline implementation_simdjson_result_base() noexcept = default; /** * Create a new error result. */ - simdjson_really_inline implementation_simdjson_result_base(error_code error) noexcept; + simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; /** * Create a new successful result. */ - simdjson_really_inline implementation_simdjson_result_base(T &&value) noexcept; + simdjson_inline implementation_simdjson_result_base(T &&value) noexcept; /** * Create a new result with both things (use if you don't want to branch when creating the result). */ - simdjson_really_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; + simdjson_inline implementation_simdjson_result_base(T &&value, error_code error) noexcept; /** * Move the value and the error to the provided variables. @@ -23332,19 +22597,19 @@ struct implementation_simdjson_result_base { * @param value The variable to assign the value to. May not be set if there is an error. * @param error The variable to assign the error to. Set to SUCCESS if there is no error. */ - simdjson_really_inline void tie(T &value, error_code &error) && noexcept; + simdjson_inline void tie(T &value, error_code &error) && noexcept; /** * Move the value to the provided variable. * * @param value The variable to assign the value to. May not be set if there is an error. */ - simdjson_really_inline error_code get(T &value) && noexcept; + simdjson_inline error_code get(T &value) && noexcept; /** * The error. */ - simdjson_really_inline error_code error() const noexcept; + simdjson_inline error_code error() const noexcept; #if SIMDJSON_EXCEPTIONS @@ -23353,28 +22618,28 @@ struct implementation_simdjson_result_base { * * @throw simdjson_error if there was an error. */ - simdjson_really_inline T& value() & noexcept(false); + simdjson_inline T& value() & noexcept(false); /** * Take the result value (move it). * * @throw simdjson_error if there was an error. */ - simdjson_really_inline T&& value() && noexcept(false); + simdjson_inline T&& value() && noexcept(false); /** * Take the result value (move it). * * @throw simdjson_error if there was an error. */ - simdjson_really_inline T&& take_value() && noexcept(false); + simdjson_inline T&& take_value() && noexcept(false); /** * Cast to the value (will throw on error). * * @throw simdjson_error if there was an error. */ - simdjson_really_inline operator T&&() && noexcept(false); + simdjson_inline operator T&&() && noexcept(false); #endif // SIMDJSON_EXCEPTIONS @@ -23383,17 +22648,17 @@ struct implementation_simdjson_result_base { * Get the result value. This function is safe if and only * the error() method returns a value that evaluates to false. */ - simdjson_really_inline const T& value_unsafe() const& noexcept; + simdjson_inline const T& value_unsafe() const& noexcept; /** * Get the result value. This function is safe if and only * the error() method returns a value that evaluates to false. */ - simdjson_really_inline T& value_unsafe() & noexcept; + simdjson_inline T& value_unsafe() & noexcept; /** * Take the result value (move it). This function is safe if and only * the error() method returns a value that evaluates to false. */ - simdjson_really_inline T&& value_unsafe() && noexcept; + simdjson_inline T&& value_unsafe() && noexcept; protected: /** users should never directly access first and second. **/ T first{}; /** Users should never directly access 'first'. **/ @@ -23457,46 +22722,46 @@ struct number { * unsigned_integer /// a positive integer larger or equal to 1<<63 * }; */ - simdjson_really_inline number_type get_number_type() const noexcept; + simdjson_inline number_type get_number_type() const noexcept; /** * return true if the automatically determined type of * the number is number_type::unsigned_integer. */ - simdjson_really_inline bool is_uint64() const noexcept; + simdjson_inline bool is_uint64() const noexcept; /** * return the value as a uint64_t, only valid if is_uint64() is true. */ - simdjson_really_inline uint64_t get_uint64() const noexcept; - simdjson_really_inline operator uint64_t() const noexcept; + simdjson_inline uint64_t get_uint64() const noexcept; + simdjson_inline operator uint64_t() const noexcept; /** * return true if the automatically determined type of * the number is number_type::signed_integer. */ - simdjson_really_inline bool is_int64() const noexcept; + simdjson_inline bool is_int64() const noexcept; /** * return the value as a int64_t, only valid if is_int64() is true. */ - simdjson_really_inline int64_t get_int64() const noexcept; - simdjson_really_inline operator int64_t() const noexcept; + simdjson_inline int64_t get_int64() const noexcept; + simdjson_inline operator int64_t() const noexcept; /** * return true if the automatically determined type of * the number is number_type::floating_point_number. */ - simdjson_really_inline bool is_double() const noexcept; + simdjson_inline bool is_double() const noexcept; /** * return the value as a double, only valid if is_double() is true. */ - simdjson_really_inline double get_double() const noexcept; - simdjson_really_inline operator double() const noexcept; + simdjson_inline double get_double() const noexcept; + simdjson_inline operator double() const noexcept; /** * Convert the number to a double. Though it always succeed, the conversion * may be lossy if the number cannot be represented exactly. */ - simdjson_really_inline double as_double() const noexcept; + simdjson_inline double as_double() const noexcept; protected: @@ -23513,13 +22778,13 @@ struct number { template friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t * src, W writer); /** Store a signed 64-bit value to the number. */ - simdjson_really_inline void append_s64(int64_t value) noexcept; + simdjson_inline void append_s64(int64_t value) noexcept; /** Store an unsigned 64-bit value to the number. */ - simdjson_really_inline void append_u64(uint64_t value) noexcept; + simdjson_inline void append_u64(uint64_t value) noexcept; /** Store a double value to the number. */ - simdjson_really_inline void append_double(double value) noexcept; + simdjson_inline void append_double(double value) noexcept; /** Specifies that the value is a double, but leave it undefined. */ - simdjson_really_inline void skip_double() noexcept; + simdjson_inline void skip_double() noexcept; /** * End of friend declarations. */ @@ -23567,10 +22832,10 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_type &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; - simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_type &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline ~simdjson_result() noexcept = default; ///< @private }; } // namespace simdjson @@ -23648,12 +22913,19 @@ class json_iterator; * * This class is deliberately simplistic and has little functionality. You can * compare a raw_json_string instance with an unescaped C string, but - * that is pretty much all you can do. + * that is nearly all you can do. + * + * The raw_json_string is unescaped. If you wish to write an unescaped version of it to your own + * buffer, you may do so using the parser.unescape(string, buff) method, using an ondemand::parser + * instance. Doing so requires you to have a sufficiently large buffer. + * + * The raw_json_string instances originate typically from field instance which in turn represent + * key-value pairs from object instances. From a field instance, you get the raw_json_string + * instance by calling key(). You can, if you want a more usable string_view instance, call + * the unescaped_key() method on the field instance. You may also create a raw_json_string from + * any other string value, with the value.get_raw_json_string() method. Again, you can get + * a more usable string_view instance by calling get_string(). * - * They originate typically from field instance which in turn represent key-value pairs from - * object instances. From a field instance, you get the raw_json_string instance by calling key(). - * You can, if you want a more usable string_view instance, call the unescaped_key() method - * on the field instance. */ class raw_json_string { public: @@ -23662,7 +22934,7 @@ class raw_json_string { * * Exists so you can declare a variable and later assign to it before use. */ - simdjson_really_inline raw_json_string() noexcept = default; + simdjson_inline raw_json_string() noexcept = default; /** * Create a new invalid raw_json_string pointed at the given location in the JSON. @@ -23671,14 +22943,14 @@ class raw_json_string { * * It *must* be terminated by a ", and be a valid JSON string. */ - simdjson_really_inline raw_json_string(const uint8_t * _buf) noexcept; + simdjson_inline raw_json_string(const uint8_t * _buf) noexcept; /** * Get the raw pointer to the beginning of the string in the JSON (just after the "). * * It is possible for this function to return a null pointer if the instance * has outlived its existence. */ - simdjson_really_inline const char * raw() const noexcept; + simdjson_inline const char * raw() const noexcept; /** * This compares the current instance to the std::string_view target: returns true if @@ -23694,7 +22966,7 @@ class raw_json_string { * Performance: the comparison may be done using memcmp which may be efficient * for long strings. */ - simdjson_really_inline bool unsafe_is_equal(size_t length, std::string_view target) const noexcept; + simdjson_inline bool unsafe_is_equal(size_t length, std::string_view target) const noexcept; /** * This compares the current instance to the std::string_view target: returns true if @@ -23711,7 +22983,7 @@ class raw_json_string { * static_assert(raw_json_string::is_free_from_unescaped_quote(target), ""); * s.unsafe_is_equal(target); */ - simdjson_really_inline bool unsafe_is_equal(std::string_view target) const noexcept; + simdjson_inline bool unsafe_is_equal(std::string_view target) const noexcept; /** * This compares the current instance to the C string target: returns true if @@ -23725,27 +22997,27 @@ class raw_json_string { * static_assert(raw_json_string::is_free_from_unescaped_quote(target), ""); * s.unsafe_is_equal(target); */ - simdjson_really_inline bool unsafe_is_equal(const char* target) const noexcept; + simdjson_inline bool unsafe_is_equal(const char* target) const noexcept; /** * This compares the current instance to the std::string_view target: returns true if * they are byte-by-byte equal (no escaping is done). */ - simdjson_really_inline bool is_equal(std::string_view target) const noexcept; + simdjson_inline bool is_equal(std::string_view target) const noexcept; /** * This compares the current instance to the C string target: returns true if * they are byte-by-byte equal (no escaping is done). */ - simdjson_really_inline bool is_equal(const char* target) const noexcept; + simdjson_inline bool is_equal(const char* target) const noexcept; /** * Returns true if target is free from unescaped quote. If target is known at * compile-time, we might expect the computation to happen at compile time with * many compilers (not all!). */ - static simdjson_really_inline bool is_free_from_unescaped_quote(std::string_view target) noexcept; - static simdjson_really_inline bool is_free_from_unescaped_quote(const char* target) noexcept; + static simdjson_inline bool is_free_from_unescaped_quote(std::string_view target) noexcept; + static simdjson_inline bool is_free_from_unescaped_quote(const char* target) noexcept; private: @@ -23754,27 +23026,13 @@ class raw_json_string { * This will set the inner pointer to zero, effectively making * this instance unusable. */ - simdjson_really_inline void consume() noexcept { buf = nullptr; } + simdjson_inline void consume() noexcept { buf = nullptr; } /** * Checks whether the inner pointer is non-null and thus usable. */ - simdjson_really_inline simdjson_warn_unused bool alive() const noexcept { return buf != nullptr; } + simdjson_inline simdjson_warn_unused bool alive() const noexcept { return buf != nullptr; } - /** - * Unescape this JSON string, replacing \\ with \, \n with newline, etc. - * - * ## IMPORTANT: string_view lifetime - * - * The string_view is only valid as long as the bytes in dst. - * - * @param dst A pointer to a buffer at least large enough to write this string as well as a \0. - * dst will be updated to the next unused location (just after the \0 written out at - * the end of this string). - * @return A string_view pointing at the unescaped string in dst - * @error STRING_ERROR if escapes are incorrect. - */ - simdjson_really_inline simdjson_warn_unused simdjson_result unescape(uint8_t *&dst) const noexcept; /** * Unescape this JSON string, replacing \\ with \, \n with newline, etc. * @@ -23784,24 +23042,25 @@ class raw_json_string { * * @param iter A json_iterator, which contains a buffer where the string will be written. */ - simdjson_really_inline simdjson_warn_unused simdjson_result unescape(json_iterator &iter) const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape(json_iterator &iter) const noexcept; const uint8_t * buf{}; friend class object; friend class field; + friend class parser; friend struct simdjson_result; }; -simdjson_unused simdjson_really_inline std::ostream &operator<<(std::ostream &, const raw_json_string &) noexcept; +simdjson_unused simdjson_inline std::ostream &operator<<(std::ostream &, const raw_json_string &) noexcept; /** * Comparisons between raw_json_string and std::string_view instances are potentially unsafe: the user is responsible * for providing a string with no unescaped quote. Note that unescaped quotes cannot be present in valid JSON strings. */ -simdjson_unused simdjson_really_inline bool operator==(const raw_json_string &a, std::string_view c) noexcept; -simdjson_unused simdjson_really_inline bool operator==(std::string_view c, const raw_json_string &a) noexcept; -simdjson_unused simdjson_really_inline bool operator!=(const raw_json_string &a, std::string_view c) noexcept; -simdjson_unused simdjson_really_inline bool operator!=(std::string_view c, const raw_json_string &a) noexcept; +simdjson_unused simdjson_inline bool operator==(const raw_json_string &a, std::string_view c) noexcept; +simdjson_unused simdjson_inline bool operator==(std::string_view c, const raw_json_string &a) noexcept; +simdjson_unused simdjson_inline bool operator!=(const raw_json_string &a, std::string_view c) noexcept; +simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_json_string &a) noexcept; } // namespace ondemand @@ -23813,14 +23072,13 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; - simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private - - simdjson_really_inline simdjson_result raw() const noexcept; - simdjson_really_inline simdjson_warn_unused simdjson_result unescape(uint8_t *&dst) const noexcept; - simdjson_really_inline simdjson_warn_unused simdjson_result unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline ~simdjson_result() noexcept = default; ///< @private + + simdjson_inline simdjson_result raw() const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept; }; } // namespace simdjson @@ -23843,20 +23101,20 @@ class token_iterator { * * Exists so you can declare a variable and later assign to it before use. */ - simdjson_really_inline token_iterator() noexcept = default; - simdjson_really_inline token_iterator(token_iterator &&other) noexcept = default; - simdjson_really_inline token_iterator &operator=(token_iterator &&other) noexcept = default; - simdjson_really_inline token_iterator(const token_iterator &other) noexcept = default; - simdjson_really_inline token_iterator &operator=(const token_iterator &other) noexcept = default; + simdjson_inline token_iterator() noexcept = default; + simdjson_inline token_iterator(token_iterator &&other) noexcept = default; + simdjson_inline token_iterator &operator=(token_iterator &&other) noexcept = default; + simdjson_inline token_iterator(const token_iterator &other) noexcept = default; + simdjson_inline token_iterator &operator=(const token_iterator &other) noexcept = default; /** * Advance to the next token (returning the current one). */ - simdjson_really_inline const uint8_t *return_current_and_advance() noexcept; + simdjson_inline const uint8_t *return_current_and_advance() noexcept; /** * Reports the current offset in bytes from the start of the underlying buffer. */ - simdjson_really_inline uint32_t current_offset() const noexcept; + simdjson_inline uint32_t current_offset() const noexcept; /** * Get the JSON text for a given token (relative). * @@ -23868,7 +23126,7 @@ class token_iterator { * TODO consider a string_view, assuming the length will get stripped out by the optimizer when * it isn't used ... */ - simdjson_really_inline const uint8_t *peek(int32_t delta=0) const noexcept; + simdjson_inline const uint8_t *peek(int32_t delta=0) const noexcept; /** * Get the maximum length of the JSON text for a given token. * @@ -23877,7 +23135,7 @@ class token_iterator { * @param delta The relative position of the token to retrieve. e.g. 0 = current token, * 1 = next token, -1 = prev token. */ - simdjson_really_inline uint32_t peek_length(int32_t delta=0) const noexcept; + simdjson_inline uint32_t peek_length(int32_t delta=0) const noexcept; /** * Get the JSON text for a given token. @@ -23887,7 +23145,7 @@ class token_iterator { * @param position The position of the token. * */ - simdjson_really_inline const uint8_t *peek(token_position position) const noexcept; + simdjson_inline const uint8_t *peek(token_position position) const noexcept; /** * Get the maximum length of the JSON text for a given token. * @@ -23895,29 +23153,29 @@ class token_iterator { * * @param position The position of the token. */ - simdjson_really_inline uint32_t peek_length(token_position position) const noexcept; + simdjson_inline uint32_t peek_length(token_position position) const noexcept; /** * Return the current index. */ - simdjson_really_inline token_position position() const noexcept; + simdjson_inline token_position position() const noexcept; /** * Reset to a previously saved index. */ - simdjson_really_inline void set_position(token_position target_position) noexcept; + simdjson_inline void set_position(token_position target_position) noexcept; // NOTE: we don't support a full C++ iterator interface, because we expect people to make // different calls to advance the iterator based on *their own* state. - simdjson_really_inline bool operator==(const token_iterator &other) const noexcept; - simdjson_really_inline bool operator!=(const token_iterator &other) const noexcept; - simdjson_really_inline bool operator>(const token_iterator &other) const noexcept; - simdjson_really_inline bool operator>=(const token_iterator &other) const noexcept; - simdjson_really_inline bool operator<(const token_iterator &other) const noexcept; - simdjson_really_inline bool operator<=(const token_iterator &other) const noexcept; + simdjson_inline bool operator==(const token_iterator &other) const noexcept; + simdjson_inline bool operator!=(const token_iterator &other) const noexcept; + simdjson_inline bool operator>(const token_iterator &other) const noexcept; + simdjson_inline bool operator>=(const token_iterator &other) const noexcept; + simdjson_inline bool operator<(const token_iterator &other) const noexcept; + simdjson_inline bool operator<=(const token_iterator &other) const noexcept; protected: - simdjson_really_inline token_iterator(const uint8_t *buf, token_position position) noexcept; + simdjson_inline token_iterator(const uint8_t *buf, token_position position) noexcept; /** * Get the index of the JSON text for a given token (relative). @@ -23927,7 +23185,7 @@ class token_iterator { * @param delta The relative position of the token to retrieve. e.g. 0 = current token, * 1 = next token, -1 = prev token. */ - simdjson_really_inline uint32_t peek_index(int32_t delta=0) const noexcept; + simdjson_inline uint32_t peek_index(int32_t delta=0) const noexcept; /** * Get the index of the JSON text for a given token. * @@ -23936,7 +23194,7 @@ class token_iterator { * @param position The position of the token. * */ - simdjson_really_inline uint32_t peek_index(token_position position) const noexcept; + simdjson_inline uint32_t peek_index(token_position position) const noexcept; const uint8_t *buf{}; token_position _position{}; @@ -23944,8 +23202,8 @@ class token_iterator { friend class json_iterator; friend class value_iterator; friend class object; - friend simdjson_really_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept; - friend simdjson_really_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept; + friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept; + friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept; }; } // namespace ondemand @@ -23957,10 +23215,10 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::token_iterator &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; - simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::token_iterator &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline ~simdjson_result() noexcept = default; ///< @private }; } // namespace simdjson @@ -24029,20 +23287,20 @@ class json_iterator { bool _streaming{false}; public: - simdjson_really_inline json_iterator() noexcept = default; - simdjson_really_inline json_iterator(json_iterator &&other) noexcept; - simdjson_really_inline json_iterator &operator=(json_iterator &&other) noexcept; - simdjson_really_inline explicit json_iterator(const json_iterator &other) noexcept = default; - simdjson_really_inline json_iterator &operator=(const json_iterator &other) noexcept = default; + simdjson_inline json_iterator() noexcept = default; + simdjson_inline json_iterator(json_iterator &&other) noexcept; + simdjson_inline json_iterator &operator=(json_iterator &&other) noexcept; + simdjson_inline explicit json_iterator(const json_iterator &other) noexcept = default; + simdjson_inline json_iterator &operator=(const json_iterator &other) noexcept = default; /** * Skips a JSON value, whether it is a scalar, array or object. */ - simdjson_warn_unused simdjson_really_inline error_code skip_child(depth_t parent_depth) noexcept; + simdjson_warn_unused simdjson_inline error_code skip_child(depth_t parent_depth) noexcept; /** * Tell whether the iterator is still at the start */ - simdjson_really_inline bool at_root() const noexcept; + simdjson_inline bool at_root() const noexcept; /** * Tell whether we should be expected to run in streaming @@ -24050,53 +23308,61 @@ class json_iterator { * that does not affect how the iterator works. It is used by * start_root_array() and start_root_object(). */ - simdjson_really_inline bool streaming() const noexcept; + simdjson_inline bool streaming() const noexcept; /** * Get the root value iterator */ - simdjson_really_inline token_position root_position() const noexcept; + simdjson_inline token_position root_position() const noexcept; /** * Assert that we are at the document depth (== 1) */ - simdjson_really_inline void assert_at_document_depth() const noexcept; + simdjson_inline void assert_at_document_depth() const noexcept; /** * Assert that we are at the root of the document */ - simdjson_really_inline void assert_at_root() const noexcept; + simdjson_inline void assert_at_root() const noexcept; /** * Tell whether the iterator is at the EOF mark */ - simdjson_really_inline bool at_end() const noexcept; + simdjson_inline bool at_end() const noexcept; /** * Tell whether the iterator is live (has not been moved). */ - simdjson_really_inline bool is_alive() const noexcept; + simdjson_inline bool is_alive() const noexcept; /** * Abandon this iterator, setting depth to 0 (as if the document is finished). */ - simdjson_really_inline void abandon() noexcept; + simdjson_inline void abandon() noexcept; /** * Advance the current token without modifying depth. */ - simdjson_really_inline const uint8_t *return_current_and_advance() noexcept; + simdjson_inline const uint8_t *return_current_and_advance() noexcept; + + /** + * Returns true if there is a single token in the index (i.e., it is + * a JSON with a scalar value such as a single number). + * + * @return whether there is a single token + */ + simdjson_inline bool is_single_token() const noexcept; /** * Assert that there are at least the given number of tokens left. * * Has no effect in release builds. */ - simdjson_really_inline void assert_more_tokens(uint32_t required_tokens=1) const noexcept; + simdjson_inline void assert_more_tokens(uint32_t required_tokens=1) const noexcept; /** * Assert that the given position addresses an actual token (is within bounds). * * Has no effect in release builds. */ - simdjson_really_inline void assert_valid_position(token_position position) const noexcept; + simdjson_inline void assert_valid_position(token_position position) const noexcept; /** * Get the JSON text for a given token (relative). * @@ -24107,7 +23373,7 @@ class json_iterator { * TODO consider a string_view, assuming the length will get stripped out by the optimizer when * it isn't used ... */ - simdjson_really_inline const uint8_t *peek(int32_t delta=0) const noexcept; + simdjson_inline const uint8_t *peek(int32_t delta=0) const noexcept; /** * Get the maximum length of the JSON text for the current token (or relative). * @@ -24115,7 +23381,7 @@ class json_iterator { * * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. */ - simdjson_really_inline uint32_t peek_length(int32_t delta=0) const noexcept; + simdjson_inline uint32_t peek_length(int32_t delta=0) const noexcept; /** * Get a pointer to the current location in the input buffer. * @@ -24124,7 +23390,7 @@ class json_iterator { * You may be pointing outside of the input buffer: it is not generally * safe to dereference this pointer. */ - simdjson_really_inline const uint8_t *unsafe_pointer() const noexcept; + simdjson_inline const uint8_t *unsafe_pointer() const noexcept; /** * Get the JSON text for a given token. * @@ -24135,7 +23401,7 @@ class json_iterator { * TODO consider a string_view, assuming the length will get stripped out by the optimizer when * it isn't used ... */ - simdjson_really_inline const uint8_t *peek(token_position position) const noexcept; + simdjson_inline const uint8_t *peek(token_position position) const noexcept; /** * Get the maximum length of the JSON text for the current token (or relative). * @@ -24143,7 +23409,7 @@ class json_iterator { * * @param position The position of the token to retrieve. */ - simdjson_really_inline uint32_t peek_length(token_position position) const noexcept; + simdjson_inline uint32_t peek_length(token_position position) const noexcept; /** * Get the JSON text for the last token in the document. * @@ -24152,7 +23418,7 @@ class json_iterator { * TODO consider a string_view, assuming the length will get stripped out by the optimizer when * it isn't used ... */ - simdjson_really_inline const uint8_t *peek_last() const noexcept; + simdjson_inline const uint8_t *peek_last() const noexcept; /** * Ascend one level. @@ -24161,7 +23427,7 @@ class json_iterator { * * @param parent_depth the expected parent depth. */ - simdjson_really_inline void ascend_to(depth_t parent_depth) noexcept; + simdjson_inline void ascend_to(depth_t parent_depth) noexcept; /** * Descend one level. @@ -24170,18 +23436,18 @@ class json_iterator { * * @param child_depth the expected child depth. */ - simdjson_really_inline void descend_to(depth_t child_depth) noexcept; - simdjson_really_inline void descend_to(depth_t child_depth, int32_t delta) noexcept; + simdjson_inline void descend_to(depth_t child_depth) noexcept; + simdjson_inline void descend_to(depth_t child_depth, int32_t delta) noexcept; /** * Get current depth. */ - simdjson_really_inline depth_t depth() const noexcept; + simdjson_inline depth_t depth() const noexcept; /** * Get current (writeable) location in the string buffer. */ - simdjson_really_inline uint8_t *&string_buf_loc() noexcept; + simdjson_inline uint8_t *&string_buf_loc() noexcept; /** * Report an unrecoverable error, preventing further iteration. @@ -24189,23 +23455,31 @@ class json_iterator { * @param error The error to report. Must not be SUCCESS, UNINITIALIZED, INCORRECT_TYPE, or NO_SUCH_FIELD. * @param message An error message to report with the error. */ - simdjson_really_inline error_code report_error(error_code error, const char *message) noexcept; + simdjson_inline error_code report_error(error_code error, const char *message) noexcept; /** * Log error, but don't stop iteration. * @param error The error to report. Must be INCORRECT_TYPE, or NO_SUCH_FIELD. * @param message An error message to report with the error. */ - simdjson_really_inline error_code optional_error(error_code error, const char *message) noexcept; + simdjson_inline error_code optional_error(error_code error, const char *message) noexcept; - template simdjson_warn_unused simdjson_really_inline bool copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept; + template simdjson_warn_unused simdjson_inline bool copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept; - simdjson_really_inline token_position position() const noexcept; - simdjson_really_inline void reenter_child(token_position position, depth_t child_depth) noexcept; -#ifdef SIMDJSON_DEVELOPMENT_CHECKS - simdjson_really_inline token_position start_position(depth_t depth) const noexcept; - simdjson_really_inline void set_start_position(depth_t depth, token_position position) noexcept; + simdjson_inline token_position position() const noexcept; + /** + * Write the raw_json_string to the string buffer and return a string_view. + * Each raw_json_string should be unescaped once, or else the string buffer might + * overflow. + */ + simdjson_inline simdjson_result unescape(raw_json_string in) noexcept; + simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept; + +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline token_position start_position(depth_t depth) const noexcept; + simdjson_inline void set_start_position(depth_t depth, token_position position) noexcept; #endif + /* Useful for debugging and logging purposes. */ inline std::string to_string() const noexcept; @@ -24219,14 +23493,21 @@ class json_iterator { * as if it had just been created. */ inline void rewind() noexcept; + /** + * This checks whether the {,},[,] are balanced so that the document + * ends with proper zero depth. This requires scanning the whole document + * and it may be expensive. It is expected that it will be rarely called. + * It does not attempt to match { with } and [ with ]. + */ + inline bool balanced() const noexcept; protected: - simdjson_really_inline json_iterator(const uint8_t *buf, ondemand::parser *parser) noexcept; + simdjson_inline json_iterator(const uint8_t *buf, ondemand::parser *parser) noexcept; /// The last token before the end - simdjson_really_inline token_position last_position() const noexcept; + simdjson_inline token_position last_position() const noexcept; /// The token *at* the end. This points at gibberish and should only be used for comparison. - simdjson_really_inline token_position end_position() const noexcept; + simdjson_inline token_position end_position() const noexcept; /// The end of the buffer. - simdjson_really_inline token_position end() const noexcept; + simdjson_inline token_position end() const noexcept; friend class document; friend class document_stream; @@ -24236,8 +23517,8 @@ class json_iterator { friend class raw_json_string; friend class parser; friend class value_iterator; - friend simdjson_really_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept; - friend simdjson_really_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept; + friend simdjson_inline void logger::log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept; + friend simdjson_inline void logger::log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept; }; // json_iterator } // namespace ondemand @@ -24249,10 +23530,10 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; + simdjson_inline simdjson_result() noexcept = default; }; } // namespace simdjson @@ -24289,61 +23570,61 @@ class value_iterator { token_position _start_position{}; public: - simdjson_really_inline value_iterator() noexcept = default; + simdjson_inline value_iterator() noexcept = default; /** * Denote that we're starting a document. */ - simdjson_really_inline void start_document() noexcept; + simdjson_inline void start_document() noexcept; /** * Skips a non-iterated or partially-iterated JSON value, whether it is a scalar, array or object. * * Optimized for scalars. */ - simdjson_warn_unused simdjson_really_inline error_code skip_child() noexcept; + simdjson_warn_unused simdjson_inline error_code skip_child() noexcept; /** * Tell whether the iterator is at the EOF mark */ - simdjson_really_inline bool at_end() const noexcept; + simdjson_inline bool at_end() const noexcept; /** * Tell whether the iterator is at the start of the value */ - simdjson_really_inline bool at_start() const noexcept; + simdjson_inline bool at_start() const noexcept; /** * Tell whether the value is open--if the value has not been used, or the array/object is still open. */ - simdjson_really_inline bool is_open() const noexcept; + simdjson_inline bool is_open() const noexcept; /** * Tell whether the value is at an object's first field (just after the {). */ - simdjson_really_inline bool at_first_field() const noexcept; + simdjson_inline bool at_first_field() const noexcept; /** * Abandon all iteration. */ - simdjson_really_inline void abandon() noexcept; + simdjson_inline void abandon() noexcept; /** * Get the child value as a value_iterator. */ - simdjson_really_inline value_iterator child_value() const noexcept; + simdjson_inline value_iterator child_value() const noexcept; /** * Get the depth of this value. */ - simdjson_really_inline int32_t depth() const noexcept; + simdjson_inline int32_t depth() const noexcept; /** * Get the JSON type of this value. * * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ - simdjson_really_inline simdjson_result type() const noexcept; + simdjson_inline simdjson_result type() const noexcept; /** * @addtogroup object Object iteration @@ -24360,7 +23641,7 @@ class value_iterator { * @returns Whether the object had any fields (returns false for empty). * @error INCORRECT_TYPE if there is no opening { */ - simdjson_warn_unused simdjson_really_inline simdjson_result start_object() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result start_object() noexcept; /** * Start an object iteration from the root. * @@ -24368,7 +23649,7 @@ class value_iterator { * @error INCORRECT_TYPE if there is no opening { * @error TAPE_ERROR if there is no matching } at end of document */ - simdjson_warn_unused simdjson_really_inline simdjson_result start_root_object() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result start_root_object() noexcept; /** * Start an object iteration after the user has already checked and moved past the {. @@ -24379,7 +23660,7 @@ class value_iterator { * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* * array or object is incomplete). */ - simdjson_warn_unused simdjson_really_inline simdjson_result started_object() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result started_object() noexcept; /** * Start an object iteration from the root, after the user has already checked and moved past the {. * @@ -24389,7 +23670,7 @@ class value_iterator { * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* * array or object is incomplete). */ - simdjson_warn_unused simdjson_really_inline simdjson_result started_root_object() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result started_root_object() noexcept; /** * Moves to the next field in an object. @@ -24401,17 +23682,17 @@ class value_iterator { * @error TAPE_ERROR If there is a comma missing between fields. * @error TAPE_ERROR If there is a comma, but not enough tokens remaining to have a key, :, and value. */ - simdjson_warn_unused simdjson_really_inline simdjson_result has_next_field() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result has_next_field() noexcept; /** * Get the current field's key. */ - simdjson_warn_unused simdjson_really_inline simdjson_result field_key() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result field_key() noexcept; /** * Pass the : in the field and move to its value. */ - simdjson_warn_unused simdjson_really_inline error_code field_value() noexcept; + simdjson_warn_unused simdjson_inline error_code field_value() noexcept; /** * Find the next field with the given key. @@ -24429,7 +23710,7 @@ class value_iterator { * unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may * fail to match some keys with escapes (\u, \n, etc.). */ - simdjson_warn_unused simdjson_really_inline error_code find_field(const std::string_view key) noexcept; + simdjson_warn_unused simdjson_inline error_code find_field(const std::string_view key) noexcept; /** * Find the next field with the given key, *without* unescaping. This assumes object order: it @@ -24448,7 +23729,7 @@ class value_iterator { * unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may * fail to match some keys with escapes (\u, \n, etc.). */ - simdjson_warn_unused simdjson_really_inline simdjson_result find_field_raw(const std::string_view key) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result find_field_raw(const std::string_view key) noexcept; /** * Find the field with the given key without regard to order, and *without* unescaping. @@ -24468,7 +23749,7 @@ class value_iterator { * unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may * fail to match some keys with escapes (\u, \n, etc.). */ - simdjson_warn_unused simdjson_really_inline simdjson_result find_field_unordered_raw(const std::string_view key) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result find_field_unordered_raw(const std::string_view key) noexcept; /** @} */ @@ -24485,7 +23766,7 @@ class value_iterator { * @returns Whether the array had any elements (returns false for empty). * @error INCORRECT_TYPE If there is no [. */ - simdjson_warn_unused simdjson_really_inline simdjson_result start_array() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result start_array() noexcept; /** * Check for an opening [ and start an array iteration while at the root. * @@ -24493,7 +23774,7 @@ class value_iterator { * @error INCORRECT_TYPE If there is no [. * @error TAPE_ERROR if there is no matching ] at end of document */ - simdjson_warn_unused simdjson_really_inline simdjson_result start_root_array() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result start_root_array() noexcept; /** * Start an array iteration, after the user has already checked and moved past the [. @@ -24504,7 +23785,7 @@ class value_iterator { * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* * array or object is incomplete). */ - simdjson_warn_unused simdjson_really_inline simdjson_result started_array() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result started_array() noexcept; /** * Start an array iteration from the root, after the user has already checked and moved past the [. * @@ -24514,7 +23795,7 @@ class value_iterator { * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* * array or object is incomplete). */ - simdjson_warn_unused simdjson_really_inline simdjson_result started_root_array() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result started_root_array() noexcept; /** * Moves to the next element in an array. @@ -24525,12 +23806,12 @@ class value_iterator { * @return Whether there is another element in the array. * @error TAPE_ERROR If there is a comma missing between elements. */ - simdjson_warn_unused simdjson_really_inline simdjson_result has_next_element() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result has_next_element() noexcept; /** * Get a child value iterator. */ - simdjson_warn_unused simdjson_really_inline value_iterator child() const noexcept; + simdjson_warn_unused simdjson_inline value_iterator child() const noexcept; /** @} */ @@ -24540,43 +23821,43 @@ class value_iterator { * @{ */ - simdjson_warn_unused simdjson_really_inline simdjson_result get_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_raw_json_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_uint64() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_uint64_in_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_int64() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_int64_in_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_double() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_double_in_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_bool() noexcept; - simdjson_really_inline bool is_null() noexcept; - simdjson_warn_unused simdjson_really_inline bool is_negative() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result is_integer() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_number_type() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_number() noexcept; - - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_raw_json_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_uint64() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_uint64_in_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_int64() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_int64_in_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_double() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_double_in_string() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_bool() noexcept; - simdjson_warn_unused simdjson_really_inline bool is_root_negative() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result is_root_integer() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_number_type() noexcept; - simdjson_warn_unused simdjson_really_inline simdjson_result get_root_number() noexcept; - simdjson_really_inline bool is_root_null() noexcept; - - simdjson_really_inline error_code error() const noexcept; - simdjson_really_inline uint8_t *&string_buf_loc() noexcept; - simdjson_really_inline const json_iterator &json_iter() const noexcept; - simdjson_really_inline json_iterator &json_iter() noexcept; - - simdjson_really_inline void assert_is_valid() const noexcept; - simdjson_really_inline bool is_valid() const noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_int64() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_double() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_bool() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_null() noexcept; + simdjson_warn_unused simdjson_inline bool is_negative() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_integer() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; + + simdjson_warn_unused simdjson_inline simdjson_result get_root_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_raw_json_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_int64() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_int64_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_double() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_double_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_bool() noexcept; + simdjson_warn_unused simdjson_inline bool is_root_negative() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_root_integer() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_number_type() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_number() noexcept; + simdjson_inline bool is_root_null() noexcept; + + simdjson_inline error_code error() const noexcept; + simdjson_inline uint8_t *&string_buf_loc() noexcept; + simdjson_inline const json_iterator &json_iter() const noexcept; + simdjson_inline json_iterator &json_iter() noexcept; + + simdjson_inline void assert_is_valid() const noexcept; + simdjson_inline bool is_valid() const noexcept; /** @} */ protected: @@ -24584,12 +23865,12 @@ class value_iterator { * Restarts an array iteration. * @returns Whether the array has any elements (returns false for empty). */ - simdjson_really_inline simdjson_result reset_array() noexcept; + simdjson_inline simdjson_result reset_array() noexcept; /** * Restarts an object iteration. * @returns Whether the object has any fields (returns false for empty). */ - simdjson_really_inline simdjson_result reset_object() noexcept; + simdjson_inline simdjson_result reset_object() noexcept; /** * move_at_start(): moves us so that we are pointing at the beginning of * the container. It updates the index so that at_start() is true and it @@ -24597,7 +23878,7 @@ class value_iterator { * * Usage: used with value::count_elements(). **/ - simdjson_really_inline void move_at_start() noexcept; + simdjson_inline void move_at_start() noexcept; /** * move_at_container_start(): moves us so that we are pointing at the beginning of @@ -24605,15 +23886,15 @@ class value_iterator { * * Usage: used with reset_array() and reset_object(). **/ - simdjson_really_inline void move_at_container_start() noexcept; + simdjson_inline void move_at_container_start() noexcept; /* Useful for debugging and logging purposes. */ inline std::string to_string() const noexcept; - simdjson_really_inline value_iterator(json_iterator *json_iter, depth_t depth, token_position start_index) noexcept; + simdjson_inline value_iterator(json_iterator *json_iter, depth_t depth, token_position start_index) noexcept; - simdjson_really_inline bool parse_null(const uint8_t *json) const noexcept; - simdjson_really_inline simdjson_result parse_bool(const uint8_t *json) const noexcept; - simdjson_really_inline const uint8_t *peek_start() const noexcept; - simdjson_really_inline uint32_t peek_start_length() const noexcept; + simdjson_inline simdjson_result parse_null(const uint8_t *json) const noexcept; + simdjson_inline simdjson_result parse_bool(const uint8_t *json) const noexcept; + simdjson_inline const uint8_t *peek_start() const noexcept; + simdjson_inline uint32_t peek_start_length() const noexcept; /** * The general idea of the advance_... methods and the peek_* methods @@ -24644,17 +23925,17 @@ class value_iterator { * Unfortunately, it makes the code more verbose, longer and maybe more error prone. */ - simdjson_really_inline void advance_scalar(const char *type) noexcept; - simdjson_really_inline void advance_root_scalar(const char *type) noexcept; - simdjson_really_inline void advance_non_root_scalar(const char *type) noexcept; + simdjson_inline void advance_scalar(const char *type) noexcept; + simdjson_inline void advance_root_scalar(const char *type) noexcept; + simdjson_inline void advance_non_root_scalar(const char *type) noexcept; - simdjson_really_inline const uint8_t *peek_scalar(const char *type) noexcept; - simdjson_really_inline const uint8_t *peek_root_scalar(const char *type) noexcept; - simdjson_really_inline const uint8_t *peek_non_root_scalar(const char *type) noexcept; + simdjson_inline const uint8_t *peek_scalar(const char *type) noexcept; + simdjson_inline const uint8_t *peek_root_scalar(const char *type) noexcept; + simdjson_inline const uint8_t *peek_non_root_scalar(const char *type) noexcept; - simdjson_really_inline error_code start_container(uint8_t start_char, const char *incorrect_type_message, const char *type) noexcept; - simdjson_really_inline error_code end_container() noexcept; + simdjson_inline error_code start_container(uint8_t start_char, const char *incorrect_type_message, const char *type) noexcept; + simdjson_inline error_code end_container() noexcept; /** * Advance to a place expecting a value (increasing depth). @@ -24662,19 +23943,19 @@ class value_iterator { * @return The current token (the one left behind). * @error TAPE_ERROR If the document ended early. */ - simdjson_really_inline simdjson_result advance_to_value() noexcept; + simdjson_inline simdjson_result advance_to_value() noexcept; - simdjson_really_inline error_code incorrect_type_error(const char *message) const noexcept; - simdjson_really_inline error_code error_unless_more_tokens(uint32_t tokens=1) const noexcept; + simdjson_inline error_code incorrect_type_error(const char *message) const noexcept; + simdjson_inline error_code error_unless_more_tokens(uint32_t tokens=1) const noexcept; - simdjson_really_inline bool is_at_start() const noexcept; + simdjson_inline bool is_at_start() const noexcept; /** * is_at_iterator_start() returns true on an array or object after it has just been * created, whether the instance is empty or not. * * Usage: used by array::begin() in debug mode (SIMDJSON_DEVELOPMENT_CHECKS) */ - simdjson_really_inline bool is_at_iterator_start() const noexcept; + simdjson_inline bool is_at_iterator_start() const noexcept; /** * Assuming that we are within an object, this returns true if we @@ -24683,7 +23964,7 @@ class value_iterator { * Usage: the skip_child() method should never be used while we are pointing * at a key inside an object. */ - simdjson_really_inline bool is_at_key() const noexcept; + simdjson_inline bool is_at_key() const noexcept; inline void assert_at_start() const noexcept; inline void assert_at_container_start() const noexcept; @@ -24693,16 +23974,16 @@ class value_iterator { inline void assert_at_non_root_start() const noexcept; /** Get the starting position of this value */ - simdjson_really_inline token_position start_position() const noexcept; + simdjson_inline token_position start_position() const noexcept; /** @copydoc error_code json_iterator::position() const noexcept; */ - simdjson_really_inline token_position position() const noexcept; + simdjson_inline token_position position() const noexcept; /** @copydoc error_code json_iterator::end_position() const noexcept; */ - simdjson_really_inline token_position last_position() const noexcept; + simdjson_inline token_position last_position() const noexcept; /** @copydoc error_code json_iterator::end_position() const noexcept; */ - simdjson_really_inline token_position end_position() const noexcept; + simdjson_inline token_position end_position() const noexcept; /** @copydoc error_code json_iterator::report_error(error_code error, const char *message) noexcept; */ - simdjson_really_inline error_code report_error(error_code error, const char *message) noexcept; + simdjson_inline error_code report_error(error_code error, const char *message) noexcept; friend class document; friend class object; @@ -24719,9 +24000,9 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value_iterator &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value_iterator &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; }; } // namespace simdjson @@ -24747,7 +24028,7 @@ class document; class array_iterator { public: /** Create a new, invalid array iterator. */ - simdjson_really_inline array_iterator() noexcept = default; + simdjson_inline array_iterator() noexcept = default; // // Iterator interface @@ -24758,7 +24039,7 @@ class array_iterator { * * Part of the std::iterator interface. */ - simdjson_really_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. + simdjson_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. /** * Check if we are at the end of the JSON. * @@ -24766,7 +24047,7 @@ class array_iterator { * * @return true if there are no more elements in the JSON array. */ - simdjson_really_inline bool operator==(const array_iterator &) const noexcept; + simdjson_inline bool operator==(const array_iterator &) const noexcept; /** * Check if there are more elements in the JSON array. * @@ -24774,18 +24055,18 @@ class array_iterator { * * @return true if there are more elements in the JSON array. */ - simdjson_really_inline bool operator!=(const array_iterator &) const noexcept; + simdjson_inline bool operator!=(const array_iterator &) const noexcept; /** * Move to the next element. * * Part of the std::iterator interface. */ - simdjson_really_inline array_iterator &operator++() noexcept; + simdjson_inline array_iterator &operator++() noexcept; private: value_iterator iter{}; - simdjson_really_inline array_iterator(const value_iterator &iter) noexcept; + simdjson_inline array_iterator(const value_iterator &iter) noexcept; friend class array; friend class value; @@ -24801,18 +24082,18 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array_iterator &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array_iterator &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; // // Iterator interface // - simdjson_really_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. - simdjson_really_inline bool operator==(const simdjson_result &) const noexcept; - simdjson_really_inline bool operator!=(const simdjson_result &) const noexcept; - simdjson_really_inline simdjson_result &operator++() noexcept; + simdjson_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. + simdjson_inline bool operator==(const simdjson_result &) const noexcept; + simdjson_inline bool operator!=(const simdjson_result &) const noexcept; + simdjson_inline simdjson_result &operator++() noexcept; }; } // namespace simdjson @@ -24832,7 +24113,7 @@ class object_iterator { * * Exists so you can declare a variable and later assign to it before use. */ - simdjson_really_inline object_iterator() noexcept = default; + simdjson_inline object_iterator() noexcept = default; // // Iterator interface @@ -24840,13 +24121,13 @@ class object_iterator { // Reads key and value, yielding them to the user. // MUST ONLY BE CALLED ONCE PER ITERATION. - simdjson_really_inline simdjson_result operator*() noexcept; + simdjson_inline simdjson_result operator*() noexcept; // Assumes it's being compared with the end. true if depth < iter->depth. - simdjson_really_inline bool operator==(const object_iterator &) const noexcept; + simdjson_inline bool operator==(const object_iterator &) const noexcept; // Assumes it's being compared with the end. true if depth >= iter->depth. - simdjson_really_inline bool operator!=(const object_iterator &) const noexcept; + simdjson_inline bool operator!=(const object_iterator &) const noexcept; // Checks for ']' and ',' - simdjson_really_inline object_iterator &operator++() noexcept; + simdjson_inline object_iterator &operator++() noexcept; private: /** @@ -24857,7 +24138,7 @@ class object_iterator { */ value_iterator iter{}; - simdjson_really_inline object_iterator(const value_iterator &iter) noexcept; + simdjson_inline object_iterator(const value_iterator &iter) noexcept; friend struct simdjson_result; friend class object; }; @@ -24871,22 +24152,22 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object_iterator &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object_iterator &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; // // Iterator interface // // Reads key and value, yielding them to the user. - simdjson_really_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. + simdjson_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. // Assumes it's being compared with the end. true if depth < iter->depth. - simdjson_really_inline bool operator==(const simdjson_result &) const noexcept; + simdjson_inline bool operator==(const simdjson_result &) const noexcept; // Assumes it's being compared with the end. true if depth >= iter->depth. - simdjson_really_inline bool operator!=(const simdjson_result &) const noexcept; + simdjson_inline bool operator!=(const simdjson_result &) const noexcept; // Checks for ']' and ',' - simdjson_really_inline simdjson_result &operator++() noexcept; + simdjson_inline simdjson_result &operator++() noexcept; }; } // namespace simdjson @@ -24910,20 +24191,20 @@ class array { * * Exists so you can declare a variable and later assign to it before use. */ - simdjson_really_inline array() noexcept = default; + simdjson_inline array() noexcept = default; /** * Begin array iteration. * * Part of the std::iterable interface. */ - simdjson_really_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result begin() noexcept; /** * Sentinel representing the end of the array. * * Part of the std::iterable interface. */ - simdjson_really_inline simdjson_result end() noexcept; + simdjson_inline simdjson_result end() noexcept; /** * This method scans the array and counts the number of elements. * The count_elements method should always be called before you have begun @@ -24938,7 +24219,7 @@ class array { * To check that an array is empty, it is more performant to use * the is_empty() method. */ - simdjson_really_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_elements() & noexcept; /** * This method scans the beginning of the array and checks whether the * array is empty. @@ -24948,7 +24229,7 @@ class array { * there is a missing comma), then an error is returned and it is no longer * safe to continue. */ - simdjson_really_inline simdjson_result is_empty() & noexcept; + simdjson_inline simdjson_result is_empty() & noexcept; /** * Reset the iterator so that we are pointing back at the * beginning of the array. You should still consume values only once even if you @@ -24993,7 +24274,7 @@ class array { * Consumes the array and returns a string_view instance corresponding to the * array as represented in JSON. It points inside the original document. */ - simdjson_really_inline simdjson_result raw_json() noexcept; + simdjson_inline simdjson_result raw_json() noexcept; /** * Get the value at the given index. This function has linear-time complexity. @@ -25002,12 +24283,12 @@ class array { * @return The value at the given index, or: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ - simdjson_really_inline simdjson_result at(size_t index) noexcept; + simdjson_inline simdjson_result at(size_t index) noexcept; protected: /** * Go to the end of the array, no matter where you are right now. */ - simdjson_really_inline error_code consume() noexcept; + simdjson_inline error_code consume() noexcept; /** * Begin array iteration. @@ -25016,7 +24297,7 @@ class array { * resulting array. * @error INCORRECT_TYPE if the iterator is not at [. */ - static simdjson_really_inline simdjson_result start(value_iterator &iter) noexcept; + static simdjson_inline simdjson_result start(value_iterator &iter) noexcept; /** * Begin array iteration from the root. * @@ -25025,7 +24306,7 @@ class array { * @error INCORRECT_TYPE if the iterator is not at [. * @error TAPE_ERROR if there is no closing ] at the end of the document. */ - static simdjson_really_inline simdjson_result start_root(value_iterator &iter) noexcept; + static simdjson_inline simdjson_result start_root(value_iterator &iter) noexcept; /** * Begin array iteration. * @@ -25034,7 +24315,7 @@ class array { * * @param iter The iterator. Must be after the initial [. Will be *moved* into the resulting array. */ - static simdjson_really_inline simdjson_result started(value_iterator &iter) noexcept; + static simdjson_inline simdjson_result started(value_iterator &iter) noexcept; /** * Create an array at the given Internal array creation. Call array::start() or array::started() instead of this. @@ -25043,7 +24324,7 @@ class array { * == true, or past the [] with is_alive() == false if the array is empty. Will be *moved* * into the resulting array. */ - simdjson_really_inline array(const value_iterator &iter) noexcept; + simdjson_inline array(const value_iterator &iter) noexcept; /** * Iterator marking current position. @@ -25068,17 +24349,17 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; - simdjson_really_inline simdjson_result begin() noexcept; - simdjson_really_inline simdjson_result end() noexcept; + simdjson_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result end() noexcept; inline simdjson_result count_elements() & noexcept; inline simdjson_result is_empty() & noexcept; inline simdjson_result reset() & noexcept; - simdjson_really_inline simdjson_result at(size_t index) noexcept; - simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at(size_t index) noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; }; } // namespace simdjson @@ -25111,11 +24392,11 @@ class document { * * Exists so you can declare a variable and later assign to it before use. */ - simdjson_really_inline document() noexcept = default; - simdjson_really_inline document(const document &other) noexcept = delete; // pass your documents by reference, not by copy - simdjson_really_inline document(document &&other) noexcept = default; - simdjson_really_inline document &operator=(const document &other) noexcept = delete; - simdjson_really_inline document &operator=(document &&other) noexcept = default; + simdjson_inline document() noexcept = default; + simdjson_inline document(const document &other) noexcept = delete; // pass your documents by reference, not by copy + simdjson_inline document(document &&other) noexcept = default; + simdjson_inline document &operator=(const document &other) noexcept = delete; + simdjson_inline document &operator=(document &&other) noexcept = default; /** * Cast this JSON value to an array. @@ -25123,49 +24404,49 @@ class document { * @returns An object that can be used to iterate the array. * @returns INCORRECT_TYPE If the JSON value is not an array. */ - simdjson_really_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_array() & noexcept; /** * Cast this JSON value to an object. * * @returns An object that can be used to look up or iterate fields. * @returns INCORRECT_TYPE If the JSON value is not an object. */ - simdjson_really_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; /** * Cast this JSON value to an unsigned integer. * * @returns A signed 64-bit integer. * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. */ - simdjson_really_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; /** * Cast this JSON value (inside string) to an unsigned integer. * * @returns A signed 64-bit integer. * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. */ - simdjson_really_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; /** * Cast this JSON value to a signed integer. * * @returns A signed 64-bit integer. * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. */ - simdjson_really_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; /** * Cast this JSON value (inside string) to a signed integer. * * @returns A signed 64-bit integer. * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. */ - simdjson_really_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; /** * Cast this JSON value to a double. * * @returns A double. * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. */ - simdjson_really_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double() noexcept; /** * Cast this JSON value (inside string) to a double. @@ -25173,7 +24454,7 @@ class document { * @returns A double. * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. */ - simdjson_really_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; /** * Cast this JSON value to a string. * @@ -25185,7 +24466,7 @@ class document { * time it parses a document or when it is destroyed. * @returns INCORRECT_TYPE if the JSON value is not a string. */ - simdjson_really_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string() noexcept; /** * Cast this JSON value to a raw_json_string. * @@ -25194,28 +24475,31 @@ class document { * @returns A pointer to the raw JSON for the given string. * @returns INCORRECT_TYPE if the JSON value is not a string. */ - simdjson_really_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; /** * Cast this JSON value to a bool. * * @returns A bool value. * @returns INCORRECT_TYPE if the JSON value is not true or false. */ - simdjson_really_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; /** * Cast this JSON value to a value when the document is an object or an array. * * @returns A value if a JSON array or object cannot be found. * @returns SCALAR_DOCUMENT_AS_VALUE error is the document is a scalar (see is_scalar() function). */ - simdjson_really_inline simdjson_result get_value() noexcept; + simdjson_inline simdjson_result get_value() noexcept; /** - * Checks if this JSON value is null. + * Checks if this JSON value is null. If and only if the value is + * null, then it is consumed (we advance). If we find a token that + * begins with 'n' but is not 'null', then an error is returned. * * @returns Whether the value is null. + * @returns INCORRECT_TYPE If the JSON value begins with 'n' and is not 'null'. */ - simdjson_really_inline bool is_null() noexcept; + simdjson_inline simdjson_result is_null() noexcept; /** * Get this value as the given type. @@ -25228,13 +24512,13 @@ class document { * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. */ - template simdjson_really_inline simdjson_result get() & noexcept { + template simdjson_inline simdjson_result get() & noexcept { // Unless the simdjson library provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library."); } /** @overload template simdjson_result get() & noexcept */ - template simdjson_really_inline simdjson_result get() && noexcept { + template simdjson_inline simdjson_result get() && noexcept { // Unless the simdjson library provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library."); @@ -25251,9 +24535,9 @@ class document { * @returns INCORRECT_TYPE If the JSON value is not an object. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ - template simdjson_really_inline error_code get(T &out) & noexcept; + template simdjson_inline error_code get(T &out) & noexcept; /** @overload template error_code get(T &out) & noexcept */ - template simdjson_really_inline error_code get(T &out) && noexcept; + template simdjson_inline error_code get(T &out) && noexcept; #if SIMDJSON_EXCEPTIONS /** @@ -25262,35 +24546,35 @@ class document { * @returns An object that can be used to iterate the array. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an array. */ - simdjson_really_inline operator array() & noexcept(false); + simdjson_inline operator array() & noexcept(false); /** * Cast this JSON value to an object. * * @returns An object that can be used to look up or iterate fields. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an object. */ - simdjson_really_inline operator object() & noexcept(false); + simdjson_inline operator object() & noexcept(false); /** * Cast this JSON value to an unsigned integer. * * @returns A signed 64-bit integer. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit unsigned integer. */ - simdjson_really_inline operator uint64_t() noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); /** * Cast this JSON value to a signed integer. * * @returns A signed 64-bit integer. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit integer. */ - simdjson_really_inline operator int64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); /** * Cast this JSON value to a double. * * @returns A double. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a valid floating-point number. */ - simdjson_really_inline operator double() noexcept(false); + simdjson_inline operator double() noexcept(false); /** * Cast this JSON value to a string. * @@ -25300,7 +24584,7 @@ class document { * time it parses a document or when it is destroyed. * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. */ - simdjson_really_inline operator std::string_view() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); /** * Cast this JSON value to a raw_json_string. * @@ -25309,21 +24593,21 @@ class document { * @returns A pointer to the raw JSON for the given string. * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. */ - simdjson_really_inline operator raw_json_string() noexcept(false); + simdjson_inline operator raw_json_string() noexcept(false); /** * Cast this JSON value to a bool. * * @returns A bool value. * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not true or false. */ - simdjson_really_inline operator bool() noexcept(false); + simdjson_inline operator bool() noexcept(false); /** * Cast this JSON value to a value. * * @returns A value value. * @exception if a JSON value cannot be found */ - simdjson_really_inline operator value() noexcept(false); + simdjson_inline operator value() noexcept(false); #endif /** * This method scans the array and counts the number of elements. @@ -25336,7 +24620,7 @@ class document { * there is a missing comma), then an error is returned and it is no longer * safe to continue. */ - simdjson_really_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_elements() & noexcept; /** * This method scans the object and counts the number of key-value pairs. * The count_fields method should always be called before you have begun @@ -25351,7 +24635,7 @@ class document { * To check that an object is empty, it is more performant to use * the is_empty() method. */ - simdjson_really_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; /** * Get the value at the given index in the array. This function has linear-time complexity. * This function should only be called once on an array instance since the array iterator is not reset between each call. @@ -25359,19 +24643,19 @@ class document { * @return The value at the given index, or: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ - simdjson_really_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; /** * Begin array iteration. * * Part of the std::iterable interface. */ - simdjson_really_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result begin() & noexcept; /** * Sentinel representing the end of the array. * * Part of the std::iterable interface. */ - simdjson_really_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result end() & noexcept; /** * Look up a field by name on an object (order-sensitive). @@ -25405,9 +24689,9 @@ class document { * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ - simdjson_really_inline simdjson_result find_field(std::string_view key) & noexcept; - /** @overload simdjson_really_inline simdjson_result find_field(std::string_view key) & noexcept; */ - simdjson_really_inline simdjson_result find_field(const char *key) & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field(const char *key) & noexcept; /** * Look up a field by name on an object, without regard to key order. @@ -25439,16 +24723,18 @@ class document { * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ - simdjson_really_inline simdjson_result find_field_unordered(const char *key) & noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ - simdjson_really_inline simdjson_result operator[](std::string_view key) & noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ - simdjson_really_inline simdjson_result operator[](const char *key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field_unordered(const char *key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](const char *key) & noexcept; /** - * Get the type of this JSON value. + * Get the type of this JSON value. It does not validate or consume the value. + * E.g., you must still call "is_null()" to check that a value is null even if + * "type()" returns json_type::null. * * NOTE: If you're only expecting a value to be one type (a typical case), it's generally * better to just call .get_double, .get_string, etc. and check for INCORRECT_TYPE (or just @@ -25456,7 +24742,7 @@ class document { * * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ - simdjson_really_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result type() noexcept; /** * Checks whether the document is a scalar (string, number, null, Boolean). @@ -25465,14 +24751,14 @@ class document { * @returns true if the type is string, number, null, Boolean * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ - simdjson_really_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; /** * Checks whether the document is a negative number. * * @returns true if the number if negative. */ - simdjson_really_inline bool is_negative() noexcept; + simdjson_inline bool is_negative() noexcept; /** * Checks whether the document is an integer number. Note that * this requires to partially parse the number string. If @@ -25482,7 +24768,7 @@ class document { * * @returns true if the number if negative. */ - simdjson_really_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; /** * Determine the number type (integer or floating-point number) as quickly * as possible. This function does not fully validate the input. It is @@ -25505,7 +24791,7 @@ class document { * * @returns the type of the number */ - simdjson_really_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; /** * Attempt to parse an ondemand::number. An ondemand::number may @@ -25533,7 +24819,7 @@ class document { * to call "get_int64()" when number.get_number_type() is not * number_type::signed_integer and when number.is_int64() is false. */ - simdjson_warn_unused simdjson_really_inline simdjson_result get_number() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; /** * Get the raw JSON for this token. @@ -25557,7 +24843,7 @@ class document { * - false * - null */ - simdjson_really_inline simdjson_result raw_json_token() noexcept; + simdjson_inline simdjson_result raw_json_token() noexcept; /** * Reset the iterator inside the document instance so we are pointing back at the @@ -25589,7 +24875,7 @@ class document { * 2 = , or } inside root array/object * 3 = key or value inside root array/object. */ - simdjson_really_inline int32_t current_depth() const noexcept; + simdjson_inline int32_t current_depth() const noexcept; /** * Get the value associated with the given JSON pointer. We use the RFC 6901 @@ -25622,26 +24908,26 @@ class document { * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed * - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function). */ - simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; /** * Consumes the document and returns a string_view instance corresponding to the * document as represented in JSON. It points inside the original byte array containing * the JSON document. */ - simdjson_really_inline simdjson_result raw_json() noexcept; + simdjson_inline simdjson_result raw_json() noexcept; protected: /** * Consumes the document. */ - simdjson_really_inline error_code consume() noexcept; + simdjson_inline error_code consume() noexcept; - simdjson_really_inline document(ondemand::json_iterator &&iter) noexcept; - simdjson_really_inline const uint8_t *text(uint32_t idx) const noexcept; + simdjson_inline document(ondemand::json_iterator &&iter) noexcept; + simdjson_inline const uint8_t *text(uint32_t idx) const noexcept; - simdjson_really_inline value_iterator resume_value_iterator() noexcept; - simdjson_really_inline value_iterator get_root_value_iterator() noexcept; - simdjson_really_inline simdjson_result start_or_resume_object() noexcept; - static simdjson_really_inline document start(ondemand::json_iterator &&iter) noexcept; + simdjson_inline value_iterator resume_value_iterator() noexcept; + simdjson_inline value_iterator get_root_value_iterator() noexcept; + simdjson_inline simdjson_result start_or_resume_object() noexcept; + static simdjson_inline document start(ondemand::json_iterator &&iter) noexcept; // // Fields @@ -25665,59 +24951,59 @@ class document { */ class document_reference { public: - simdjson_really_inline document_reference() noexcept; - simdjson_really_inline document_reference(document &d) noexcept; - simdjson_really_inline document_reference(const document_reference &other) noexcept = default; - simdjson_really_inline document_reference& operator=(const document_reference &other) noexcept = default; - simdjson_really_inline void rewind() noexcept; - simdjson_really_inline simdjson_result get_array() & noexcept; - simdjson_really_inline simdjson_result get_object() & noexcept; - simdjson_really_inline simdjson_result get_uint64() noexcept; - simdjson_really_inline simdjson_result get_int64() noexcept; - simdjson_really_inline simdjson_result get_double() noexcept; - simdjson_really_inline simdjson_result get_string() noexcept; - simdjson_really_inline simdjson_result get_raw_json_string() noexcept; - simdjson_really_inline simdjson_result get_bool() noexcept; - simdjson_really_inline simdjson_result get_value() noexcept; - - simdjson_really_inline bool is_null() noexcept; - simdjson_really_inline simdjson_result raw_json() noexcept; - simdjson_really_inline operator document&() const noexcept; + simdjson_inline document_reference() noexcept; + simdjson_inline document_reference(document &d) noexcept; + simdjson_inline document_reference(const document_reference &other) noexcept = default; + simdjson_inline document_reference& operator=(const document_reference &other) noexcept = default; + simdjson_inline void rewind() noexcept; + simdjson_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_value() noexcept; + + simdjson_inline simdjson_result is_null() noexcept; + simdjson_inline simdjson_result raw_json() noexcept; + simdjson_inline operator document&() const noexcept; #if SIMDJSON_EXCEPTIONS - simdjson_really_inline operator array() & noexcept(false); - simdjson_really_inline operator object() & noexcept(false); - simdjson_really_inline operator uint64_t() noexcept(false); - simdjson_really_inline operator int64_t() noexcept(false); - simdjson_really_inline operator double() noexcept(false); - simdjson_really_inline operator std::string_view() noexcept(false); - simdjson_really_inline operator raw_json_string() noexcept(false); - simdjson_really_inline operator bool() noexcept(false); - simdjson_really_inline operator value() noexcept(false); + simdjson_inline operator array() & noexcept(false); + simdjson_inline operator object() & noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); + simdjson_inline operator value() noexcept(false); #endif - simdjson_really_inline simdjson_result count_elements() & noexcept; - simdjson_really_inline simdjson_result count_fields() & noexcept; - simdjson_really_inline simdjson_result at(size_t index) & noexcept; - simdjson_really_inline simdjson_result begin() & noexcept; - simdjson_really_inline simdjson_result end() & noexcept; - simdjson_really_inline simdjson_result find_field(std::string_view key) & noexcept; - simdjson_really_inline simdjson_result find_field(const char *key) & noexcept; - simdjson_really_inline simdjson_result operator[](std::string_view key) & noexcept; - simdjson_really_inline simdjson_result operator[](const char *key) & noexcept; - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; - simdjson_really_inline simdjson_result find_field_unordered(const char *key) & noexcept; - - simdjson_really_inline simdjson_result type() noexcept; - simdjson_really_inline simdjson_result is_scalar() noexcept; - - simdjson_really_inline simdjson_result current_location() noexcept; - simdjson_really_inline int32_t current_depth() const noexcept; - simdjson_really_inline bool is_negative() noexcept; - simdjson_really_inline simdjson_result is_integer() noexcept; - simdjson_really_inline simdjson_result get_number_type() noexcept; - simdjson_really_inline simdjson_result get_number() noexcept; - simdjson_really_inline simdjson_result raw_json_token() noexcept; - simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(const char *key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](const char *key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(const char *key) & noexcept; + + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + + simdjson_inline simdjson_result current_location() noexcept; + simdjson_inline int32_t current_depth() const noexcept; + simdjson_inline bool is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + simdjson_inline simdjson_result raw_json_token() noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; private: document *doc{nullptr}; }; @@ -25730,63 +25016,63 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; - simdjson_really_inline error_code rewind() noexcept; - - simdjson_really_inline simdjson_result get_array() & noexcept; - simdjson_really_inline simdjson_result get_object() & noexcept; - simdjson_really_inline simdjson_result get_uint64() noexcept; - simdjson_really_inline simdjson_result get_int64() noexcept; - simdjson_really_inline simdjson_result get_double() noexcept; - simdjson_really_inline simdjson_result get_double_from_string() noexcept; - simdjson_really_inline simdjson_result get_string() noexcept; - simdjson_really_inline simdjson_result get_raw_json_string() noexcept; - simdjson_really_inline simdjson_result get_bool() noexcept; - simdjson_really_inline simdjson_result get_value() noexcept; - simdjson_really_inline bool is_null() noexcept; - - template simdjson_really_inline simdjson_result get() & noexcept; - template simdjson_really_inline simdjson_result get() && noexcept; - - template simdjson_really_inline error_code get(T &out) & noexcept; - template simdjson_really_inline error_code get(T &out) && noexcept; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline error_code rewind() noexcept; + + simdjson_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_from_string() noexcept; + simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_value() noexcept; + simdjson_inline simdjson_result is_null() noexcept; + + template simdjson_inline simdjson_result get() & noexcept; + template simdjson_inline simdjson_result get() && noexcept; + + template simdjson_inline error_code get(T &out) & noexcept; + template simdjson_inline error_code get(T &out) && noexcept; #if SIMDJSON_EXCEPTIONS - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() & noexcept(false); - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() & noexcept(false); - simdjson_really_inline operator uint64_t() noexcept(false); - simdjson_really_inline operator int64_t() noexcept(false); - simdjson_really_inline operator double() noexcept(false); - simdjson_really_inline operator std::string_view() noexcept(false); - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false); - simdjson_really_inline operator bool() noexcept(false); - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value() noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() & noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() & noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value() noexcept(false); #endif - simdjson_really_inline simdjson_result count_elements() & noexcept; - simdjson_really_inline simdjson_result count_fields() & noexcept; - simdjson_really_inline simdjson_result at(size_t index) & noexcept; - simdjson_really_inline simdjson_result begin() & noexcept; - simdjson_really_inline simdjson_result end() & noexcept; - simdjson_really_inline simdjson_result find_field(std::string_view key) & noexcept; - simdjson_really_inline simdjson_result find_field(const char *key) & noexcept; - simdjson_really_inline simdjson_result operator[](std::string_view key) & noexcept; - simdjson_really_inline simdjson_result operator[](const char *key) & noexcept; - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; - simdjson_really_inline simdjson_result find_field_unordered(const char *key) & noexcept; - simdjson_really_inline simdjson_result type() noexcept; - simdjson_really_inline simdjson_result is_scalar() noexcept; - simdjson_really_inline simdjson_result current_location() noexcept; - simdjson_really_inline int32_t current_depth() const noexcept; - simdjson_really_inline bool is_negative() noexcept; - simdjson_really_inline simdjson_result is_integer() noexcept; - simdjson_really_inline simdjson_result get_number_type() noexcept; - simdjson_really_inline simdjson_result get_number() noexcept; - /** @copydoc simdjson_really_inline std::string_view document::raw_json_token() const noexcept */ - simdjson_really_inline simdjson_result raw_json_token() noexcept; - - simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(const char *key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](const char *key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(const char *key) & noexcept; + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result current_location() noexcept; + simdjson_inline int32_t current_depth() const noexcept; + simdjson_inline bool is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + /** @copydoc simdjson_inline std::string_view document::raw_json_token() const noexcept */ + simdjson_inline simdjson_result raw_json_token() noexcept; + + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; }; @@ -25799,55 +25085,55 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_reference value, error_code error) noexcept; - simdjson_really_inline simdjson_result() noexcept = default; - simdjson_really_inline error_code rewind() noexcept; - - simdjson_really_inline simdjson_result get_array() & noexcept; - simdjson_really_inline simdjson_result get_object() & noexcept; - simdjson_really_inline simdjson_result get_uint64() noexcept; - simdjson_really_inline simdjson_result get_int64() noexcept; - simdjson_really_inline simdjson_result get_double() noexcept; - simdjson_really_inline simdjson_result get_string() noexcept; - simdjson_really_inline simdjson_result get_raw_json_string() noexcept; - simdjson_really_inline simdjson_result get_bool() noexcept; - simdjson_really_inline simdjson_result get_value() noexcept; - simdjson_really_inline bool is_null() noexcept; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_reference value, error_code error) noexcept; + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline error_code rewind() noexcept; + + simdjson_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_value() noexcept; + simdjson_inline simdjson_result is_null() noexcept; #if SIMDJSON_EXCEPTIONS - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() & noexcept(false); - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() & noexcept(false); - simdjson_really_inline operator uint64_t() noexcept(false); - simdjson_really_inline operator int64_t() noexcept(false); - simdjson_really_inline operator double() noexcept(false); - simdjson_really_inline operator std::string_view() noexcept(false); - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false); - simdjson_really_inline operator bool() noexcept(false); - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value() noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() & noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() & noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value() noexcept(false); #endif - simdjson_really_inline simdjson_result count_elements() & noexcept; - simdjson_really_inline simdjson_result count_fields() & noexcept; - simdjson_really_inline simdjson_result at(size_t index) & noexcept; - simdjson_really_inline simdjson_result begin() & noexcept; - simdjson_really_inline simdjson_result end() & noexcept; - simdjson_really_inline simdjson_result find_field(std::string_view key) & noexcept; - simdjson_really_inline simdjson_result find_field(const char *key) & noexcept; - simdjson_really_inline simdjson_result operator[](std::string_view key) & noexcept; - simdjson_really_inline simdjson_result operator[](const char *key) & noexcept; - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; - simdjson_really_inline simdjson_result find_field_unordered(const char *key) & noexcept; - simdjson_really_inline simdjson_result type() noexcept; - simdjson_really_inline simdjson_result is_scalar() noexcept; - simdjson_really_inline simdjson_result current_location() noexcept; - simdjson_really_inline int32_t current_depth() const noexcept; - simdjson_really_inline bool is_negative() noexcept; - simdjson_really_inline simdjson_result is_integer() noexcept; - simdjson_really_inline simdjson_result get_number_type() noexcept; - simdjson_really_inline simdjson_result get_number() noexcept; - /** @copydoc simdjson_really_inline std::string_view document_reference::raw_json_token() const noexcept */ - simdjson_really_inline simdjson_result raw_json_token() noexcept; - - simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(const char *key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](const char *key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(const char *key) & noexcept; + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result current_location() noexcept; + simdjson_inline simdjson_result current_depth() const noexcept; + simdjson_inline simdjson_result is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + /** @copydoc simdjson_inline std::string_view document_reference::raw_json_token() const noexcept */ + simdjson_inline simdjson_result raw_json_token() noexcept; + + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; }; @@ -25866,7 +25152,8 @@ class object; class raw_json_string; /** - * An ephemeral JSON value returned during iteration. + * An ephemeral JSON value returned during iteration. It is only valid for as long as you do + * not access more data in the JSON document. */ class value { public: @@ -25875,7 +25162,7 @@ class value { * * Exists so you can declare a variable and later assign to it before use. */ - simdjson_really_inline value() noexcept = default; + simdjson_inline value() noexcept = default; /** * Get this value as the given type. @@ -25888,7 +25175,7 @@ class value { * @returns A value of the given type, parsed from the JSON. * @returns INCORRECT_TYPE If the JSON value is not the given type. */ - template simdjson_really_inline simdjson_result get() noexcept { + template simdjson_inline simdjson_result get() noexcept { // Unless the simdjson library provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library."); @@ -25903,7 +25190,7 @@ class value { * @returns INCORRECT_TYPE If the JSON value is not an object. * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. */ - template simdjson_really_inline error_code get(T &out) noexcept; + template simdjson_inline error_code get(T &out) noexcept; /** * Cast this JSON value to an array. @@ -25911,7 +25198,7 @@ class value { * @returns An object that can be used to iterate the array. * @returns INCORRECT_TYPE If the JSON value is not an array. */ - simdjson_really_inline simdjson_result get_array() noexcept; + simdjson_inline simdjson_result get_array() noexcept; /** * Cast this JSON value to an object. @@ -25919,7 +25206,7 @@ class value { * @returns An object that can be used to look up or iterate fields. * @returns INCORRECT_TYPE If the JSON value is not an object. */ - simdjson_really_inline simdjson_result get_object() noexcept; + simdjson_inline simdjson_result get_object() noexcept; /** * Cast this JSON value to an unsigned integer. @@ -25927,7 +25214,7 @@ class value { * @returns A unsigned 64-bit integer. * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. */ - simdjson_really_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; /** * Cast this JSON value (inside string) to a unsigned integer. @@ -25935,7 +25222,7 @@ class value { * @returns A unsigned 64-bit integer. * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. */ - simdjson_really_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; /** * Cast this JSON value to a signed integer. @@ -25943,7 +25230,7 @@ class value { * @returns A signed 64-bit integer. * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. */ - simdjson_really_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; /** * Cast this JSON value (inside string) to a signed integer. @@ -25951,7 +25238,7 @@ class value { * @returns A signed 64-bit integer. * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. */ - simdjson_really_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; /** * Cast this JSON value to a double. @@ -25959,7 +25246,7 @@ class value { * @returns A double. * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. */ - simdjson_really_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double() noexcept; /** * Cast this JSON value (inside string) to a double @@ -25967,7 +25254,7 @@ class value { * @returns A double. * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. */ - simdjson_really_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; /** * Cast this JSON value to a string. @@ -25983,7 +25270,7 @@ class value { * time it parses a document or when it is destroyed. * @returns INCORRECT_TYPE if the JSON value is not a string. */ - simdjson_really_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_string() noexcept; /** * Cast this JSON value to a raw_json_string. @@ -25993,7 +25280,7 @@ class value { * @returns A pointer to the raw JSON for the given string. * @returns INCORRECT_TYPE if the JSON value is not a string. */ - simdjson_really_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; /** * Cast this JSON value to a bool. @@ -26001,14 +25288,17 @@ class value { * @returns A bool value. * @returns INCORRECT_TYPE if the JSON value is not true or false. */ - simdjson_really_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; /** - * Checks if this JSON value is null. + * Checks if this JSON value is null. If and only if the value is + * null, then it is consumed (we advance). If we find a token that + * begins with 'n' but is not 'null', then an error is returned. * * @returns Whether the value is null. + * @returns INCORRECT_TYPE If the JSON value begins with 'n' and is not 'null'. */ - simdjson_really_inline bool is_null() noexcept; + simdjson_inline simdjson_result is_null() noexcept; #if SIMDJSON_EXCEPTIONS /** @@ -26017,35 +25307,35 @@ class value { * @returns An object that can be used to iterate the array. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an array. */ - simdjson_really_inline operator array() noexcept(false); + simdjson_inline operator array() noexcept(false); /** * Cast this JSON value to an object. * * @returns An object that can be used to look up or iterate fields. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an object. */ - simdjson_really_inline operator object() noexcept(false); + simdjson_inline operator object() noexcept(false); /** * Cast this JSON value to an unsigned integer. * * @returns A signed 64-bit integer. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit unsigned integer. */ - simdjson_really_inline operator uint64_t() noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); /** * Cast this JSON value to a signed integer. * * @returns A signed 64-bit integer. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit integer. */ - simdjson_really_inline operator int64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); /** * Cast this JSON value to a double. * * @returns A double. * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a valid floating-point number. */ - simdjson_really_inline operator double() noexcept(false); + simdjson_inline operator double() noexcept(false); /** * Cast this JSON value to a string. * @@ -26057,7 +25347,7 @@ class value { * time it parses a document or when it is destroyed. * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. */ - simdjson_really_inline operator std::string_view() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); /** * Cast this JSON value to a raw_json_string. * @@ -26066,14 +25356,14 @@ class value { * @returns A pointer to the raw JSON for the given string. * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. */ - simdjson_really_inline operator raw_json_string() noexcept(false); + simdjson_inline operator raw_json_string() noexcept(false); /** * Cast this JSON value to a bool. * * @returns A bool value. * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not true or false. */ - simdjson_really_inline operator bool() noexcept(false); + simdjson_inline operator bool() noexcept(false); #endif /** @@ -26083,13 +25373,13 @@ class value { * * @returns INCORRECT_TYPE If the JSON value is not an array. */ - simdjson_really_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result begin() & noexcept; /** * Sentinel representing the end of the array. * * Part of the std::iterable interface. */ - simdjson_really_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result end() & noexcept; /** * This method scans the array and counts the number of elements. * The count_elements method should always be called before you have begun @@ -26100,8 +25390,11 @@ class value { * beginning as if it had never been accessed. If the JSON is malformed (e.g., * there is a missing comma), then an error is returned and it is no longer * safe to continue. + * + * Performance hint: You should only call count_elements() as a last + * resort as it may require scanning the document twice or more. */ - simdjson_really_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_elements() & noexcept; /** * This method scans the object and counts the number of key-value pairs. * The count_fields method should always be called before you have begun @@ -26115,8 +25408,11 @@ class value { * * To check that an object is empty, it is more performant to use * the is_empty() method on the object instance. + * + * Performance hint: You should only call count_fields() as a last + * resort as it may require scanning the document twice or more. */ - simdjson_really_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; /** * Get the value at the given index in the array. This function has linear-time complexity. * This function should only be called once on an array instance since the array iterator is not reset between each call. @@ -26124,7 +25420,7 @@ class value { * @return The value at the given index, or: * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length */ - simdjson_really_inline simdjson_result at(size_t index) noexcept; + simdjson_inline simdjson_result at(size_t index) noexcept; /** * Look up a field by name on an object (order-sensitive). * @@ -26147,9 +25443,9 @@ class value { * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ - simdjson_really_inline simdjson_result find_field(std::string_view key) noexcept; - /** @overload simdjson_really_inline simdjson_result find_field(std::string_view key) noexcept; */ - simdjson_really_inline simdjson_result find_field(const char *key) noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field(const char *key) noexcept; /** * Look up a field by name on an object, without regard to key order. @@ -26173,16 +25469,18 @@ class value { * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ - simdjson_really_inline simdjson_result find_field_unordered(const char *key) noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ - simdjson_really_inline simdjson_result operator[](std::string_view key) noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ - simdjson_really_inline simdjson_result operator[](const char *key) noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field_unordered(const char *key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](const char *key) noexcept; /** - * Get the type of this JSON value. + * Get the type of this JSON value. It does not validate or consume the value. + * E.g., you must still call "is_null()" to check that a value is null even if + * "type()" returns json_type::null. * * NOTE: If you're only expecting a value to be one type (a typical case), it's generally * better to just call .get_double, .get_string, etc. and check for INCORRECT_TYPE (or just @@ -26192,7 +25490,7 @@ class value { * json_type::number, json_type::boolean, or json_type::null). * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ - simdjson_really_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result type() noexcept; /** * Checks whether the value is a scalar (string, number, null, Boolean). @@ -26201,14 +25499,14 @@ class value { * @returns true if the type is string, number, null, Boolean * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ - simdjson_really_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; /** * Checks whether the value is a negative number. * * @returns true if the number if negative. */ - simdjson_really_inline bool is_negative() noexcept; + simdjson_inline bool is_negative() noexcept; /** * Checks whether the value is an integer number. Note that * this requires to partially parse the number string. If @@ -26222,7 +25520,7 @@ class value { * * @returns true if the number if negative. */ - simdjson_really_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; /** * Determine the number type (integer or floating-point number) as quickly * as possible. This function does not fully validate the input. It is @@ -26245,7 +25543,7 @@ class value { * * @returns the type of the number */ - simdjson_really_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; /** * Attempt to parse an ondemand::number. An ondemand::number may @@ -26277,7 +25575,7 @@ class value { * calling 'get_number()', you scan the number string only once, determining * efficiently the type and storing it in an efficient manner. */ - simdjson_warn_unused simdjson_really_inline simdjson_result get_number() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; /** @@ -26303,12 +25601,12 @@ class value { * - false * - null */ - simdjson_really_inline std::string_view raw_json_token() noexcept; + simdjson_inline std::string_view raw_json_token() noexcept; /** * Returns the current location in the document if in bounds. */ - simdjson_really_inline simdjson_result current_location() noexcept; + simdjson_inline simdjson_result current_location() noexcept; /** * Returns the current depth in the document if in bounds. @@ -26319,7 +25617,7 @@ class value { * 2 = , or } inside root array/object * 3 = key or value inside root array/object. */ - simdjson_really_inline int32_t current_depth() const noexcept; + simdjson_inline int32_t current_depth() const noexcept; /** * Get the value associated with the given JSON pointer. We use the RFC 6901 @@ -26363,38 +25661,38 @@ class value { * - INCORRECT_TYPE if a non-integer is used to access an array * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed */ - simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; protected: /** * Create a value. */ - simdjson_really_inline value(const value_iterator &iter) noexcept; + simdjson_inline value(const value_iterator &iter) noexcept; /** * Skip this value, allowing iteration to continue. */ - simdjson_really_inline void skip() noexcept; + simdjson_inline void skip() noexcept; /** * Start a value at the current position. * * (It should already be started; this is just a self-documentation method.) */ - static simdjson_really_inline value start(const value_iterator &iter) noexcept; + static simdjson_inline value start(const value_iterator &iter) noexcept; /** * Resume a value. */ - static simdjson_really_inline value resume(const value_iterator &iter) noexcept; + static simdjson_inline value resume(const value_iterator &iter) noexcept; /** * Get the object, starting or resuming it as necessary */ - simdjson_really_inline simdjson_result start_or_resume_object() noexcept; + simdjson_inline simdjson_result start_or_resume_object() noexcept; - // simdjson_really_inline void log_value(const char *type) const noexcept; - // simdjson_really_inline void log_error(const char *message) const noexcept; + // simdjson_inline void log_value(const char *type) const noexcept; + // simdjson_inline void log_error(const char *message) const noexcept; value_iterator iter{}; @@ -26415,43 +25713,43 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; - simdjson_really_inline simdjson_result get_array() noexcept; - simdjson_really_inline simdjson_result get_object() noexcept; + simdjson_inline simdjson_result get_array() noexcept; + simdjson_inline simdjson_result get_object() noexcept; - simdjson_really_inline simdjson_result get_uint64() noexcept; - simdjson_really_inline simdjson_result get_uint64_in_string() noexcept; - simdjson_really_inline simdjson_result get_int64() noexcept; - simdjson_really_inline simdjson_result get_int64_in_string() noexcept; - simdjson_really_inline simdjson_result get_double() noexcept; - simdjson_really_inline simdjson_result get_double_in_string() noexcept; - simdjson_really_inline simdjson_result get_string() noexcept; - simdjson_really_inline simdjson_result get_raw_json_string() noexcept; - simdjson_really_inline simdjson_result get_bool() noexcept; - simdjson_really_inline bool is_null() noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result is_null() noexcept; - template simdjson_really_inline simdjson_result get() noexcept; + template simdjson_inline simdjson_result get() noexcept; - template simdjson_really_inline error_code get(T &out) noexcept; + template simdjson_inline error_code get(T &out) noexcept; #if SIMDJSON_EXCEPTIONS - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() noexcept(false); - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() noexcept(false); - simdjson_really_inline operator uint64_t() noexcept(false); - simdjson_really_inline operator int64_t() noexcept(false); - simdjson_really_inline operator double() noexcept(false); - simdjson_really_inline operator std::string_view() noexcept(false); - simdjson_really_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false); - simdjson_really_inline operator bool() noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); #endif - simdjson_really_inline simdjson_result count_elements() & noexcept; - simdjson_really_inline simdjson_result count_fields() & noexcept; - simdjson_really_inline simdjson_result at(size_t index) noexcept; - simdjson_really_inline simdjson_result begin() & noexcept; - simdjson_really_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; /** * Look up a field by name on an object (order-sensitive). @@ -26473,9 +25771,9 @@ struct simdjson_result : publi * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ - simdjson_really_inline simdjson_result find_field(std::string_view key) noexcept; - /** @overload simdjson_really_inline simdjson_result find_field(std::string_view key) noexcept; */ - simdjson_really_inline simdjson_result find_field(const char *key) noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field(const char *key) noexcept; /** * Look up a field by name on an object, without regard to key order. @@ -26496,13 +25794,13 @@ struct simdjson_result : publi * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ - simdjson_really_inline simdjson_result find_field_unordered(const char *key) noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ - simdjson_really_inline simdjson_result operator[](std::string_view key) noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ - simdjson_really_inline simdjson_result operator[](const char *key) noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field_unordered(const char *key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](const char *key) noexcept; /** * Get the type of this JSON value. @@ -26511,21 +25809,21 @@ struct simdjson_result : publi * better to just call .get_double, .get_string, etc. and check for INCORRECT_TYPE (or just * let it throw an exception). */ - simdjson_really_inline simdjson_result type() noexcept; - simdjson_really_inline simdjson_result is_scalar() noexcept; - simdjson_really_inline simdjson_result is_negative() noexcept; - simdjson_really_inline simdjson_result is_integer() noexcept; - simdjson_really_inline simdjson_result get_number_type() noexcept; - simdjson_really_inline simdjson_result get_number() noexcept; - - /** @copydoc simdjson_really_inline std::string_view value::raw_json_token() const noexcept */ - simdjson_really_inline simdjson_result raw_json_token() noexcept; - - /** @copydoc simdjson_really_inline simdjson_result current_location() noexcept */ - simdjson_really_inline simdjson_result current_location() noexcept; - /** @copydoc simdjson_really_inline int32_t current_depth() const noexcept */ - simdjson_really_inline int32_t current_depth() const noexcept; - simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + + /** @copydoc simdjson_inline std::string_view value::raw_json_token() const noexcept */ + simdjson_inline simdjson_result raw_json_token() noexcept; + + /** @copydoc simdjson_inline simdjson_result current_location() noexcept */ + simdjson_inline simdjson_result current_location() noexcept; + /** @copydoc simdjson_inline int32_t current_depth() const noexcept */ + simdjson_inline simdjson_result current_depth() const noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; }; } // namespace simdjson @@ -26550,7 +25848,7 @@ class field : public std::pair { * * Exists so you can declare a variable and later assign to it before use. */ - simdjson_really_inline field() noexcept; + simdjson_inline field() noexcept; /** * Get the key as a string_view (for higher speed, consider raw_key). @@ -26560,25 +25858,25 @@ class field : public std::pair { * This consumes the key: once you have called unescaped_key(), you cannot * call it again nor can you call key(). */ - simdjson_really_inline simdjson_warn_unused simdjson_result unescaped_key() noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key() noexcept; /** * Get the key as a raw_json_string. Can be used for direct comparison with * an unescaped C string: e.g., key() == "test". */ - simdjson_really_inline raw_json_string key() const noexcept; + simdjson_inline raw_json_string key() const noexcept; /** * Get the field value. */ - simdjson_really_inline ondemand::value &value() & noexcept; + simdjson_inline ondemand::value &value() & noexcept; /** * @overload ondemand::value &ondemand::value() & noexcept */ - simdjson_really_inline ondemand::value value() && noexcept; + simdjson_inline ondemand::value value() && noexcept; protected: - simdjson_really_inline field(raw_json_string key, ondemand::value &&value) noexcept; - static simdjson_really_inline simdjson_result start(value_iterator &parent_iter) noexcept; - static simdjson_really_inline simdjson_result start(const value_iterator &parent_iter, raw_json_string key) noexcept; + simdjson_inline field(raw_json_string key, ondemand::value &&value) noexcept; + static simdjson_inline simdjson_result start(value_iterator &parent_iter) noexcept; + static simdjson_inline simdjson_result start(const value_iterator &parent_iter, raw_json_string key) noexcept; friend struct simdjson_result; friend class object_iterator; }; @@ -26592,13 +25890,13 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::field &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::field &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; - simdjson_really_inline simdjson_result unescaped_key() noexcept; - simdjson_really_inline simdjson_result key() noexcept; - simdjson_really_inline simdjson_result value() noexcept; + simdjson_inline simdjson_result unescaped_key() noexcept; + simdjson_inline simdjson_result key() noexcept; + simdjson_inline simdjson_result value() noexcept; }; } // namespace simdjson @@ -26619,10 +25917,10 @@ class object { * * Exists so you can declare a variable and later assign to it before use. */ - simdjson_really_inline object() noexcept = default; + simdjson_inline object() noexcept = default; - simdjson_really_inline simdjson_result begin() noexcept; - simdjson_really_inline simdjson_result end() noexcept; + simdjson_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result end() noexcept; /** * Look up a field by name on an object (order-sensitive). * @@ -26643,7 +25941,8 @@ class object { * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. * * You must consume the fields on an object one at a time. A request for a new key - * invalidates previous field values: it makes them unsafe. E.g., the array + * invalidates previous field values: it makes them unsafe. The value instance you get + * from `content["bids"]` becomes invalid when you call `content["asks"]`. The array * given by content["bids"].get_array() should not be accessed after you have called * content["asks"].get_array(). You can detect such mistakes by first compiling and running * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an @@ -26656,9 +25955,9 @@ class object { * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ - simdjson_really_inline simdjson_result find_field(std::string_view key) & noexcept; - /** @overload simdjson_really_inline simdjson_result find_field(std::string_view key) & noexcept; */ - simdjson_really_inline simdjson_result find_field(std::string_view key) && noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field(std::string_view key) && noexcept; /** * Look up a field by name on an object, without regard to key order. @@ -26680,7 +25979,8 @@ class object { * that only one field is returned. * * You must consume the fields on an object one at a time. A request for a new key - * invalidates previous field values: it makes them unsafe. E.g., the array + * invalidates previous field values: it makes them unsafe. The value instance you get + * from `content["bids"]` becomes invalid when you call `content["asks"]`. The array * given by content["bids"].get_array() should not be accessed after you have called * content["asks"].get_array(). You can detect such mistakes by first compiling and running * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an @@ -26692,13 +25992,13 @@ class object { * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) && noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ - simdjson_really_inline simdjson_result operator[](std::string_view key) & noexcept; - /** @overload simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ - simdjson_really_inline simdjson_result operator[](std::string_view key) && noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) && noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; /** * Get the value associated with the given JSON pointer. We use the RFC 6901 @@ -26769,27 +26069,30 @@ class object { * * To check that an object is empty, it is more performant to use * the is_empty() method. + * + * Performance hint: You should only call count_fields() as a last + * resort as it may require scanning the document twice or more. */ - simdjson_really_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; /** * Consumes the object and returns a string_view instance corresponding to the * object as represented in JSON. It points inside the original byte array containing * the JSON document. */ - simdjson_really_inline simdjson_result raw_json() noexcept; + simdjson_inline simdjson_result raw_json() noexcept; protected: /** * Go to the end of the object, no matter where you are right now. */ - simdjson_really_inline error_code consume() noexcept; - static simdjson_really_inline simdjson_result start(value_iterator &iter) noexcept; - static simdjson_really_inline simdjson_result start_root(value_iterator &iter) noexcept; - static simdjson_really_inline simdjson_result started(value_iterator &iter) noexcept; - static simdjson_really_inline object resume(const value_iterator &iter) noexcept; - simdjson_really_inline object(const value_iterator &iter) noexcept; + simdjson_inline error_code consume() noexcept; + static simdjson_inline simdjson_result start(value_iterator &iter) noexcept; + static simdjson_inline simdjson_result start_root(value_iterator &iter) noexcept; + static simdjson_inline simdjson_result started(value_iterator &iter) noexcept; + static simdjson_inline object resume(const value_iterator &iter) noexcept; + simdjson_inline object(const value_iterator &iter) noexcept; - simdjson_warn_unused simdjson_really_inline error_code find_field_raw(const std::string_view key) noexcept; + simdjson_warn_unused simdjson_inline error_code find_field_raw(const std::string_view key) noexcept; value_iterator iter{}; @@ -26807,19 +26110,19 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; - - simdjson_really_inline simdjson_result begin() noexcept; - simdjson_really_inline simdjson_result end() noexcept; - simdjson_really_inline simdjson_result find_field(std::string_view key) & noexcept; - simdjson_really_inline simdjson_result find_field(std::string_view key) && noexcept; - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; - simdjson_really_inline simdjson_result find_field_unordered(std::string_view key) && noexcept; - simdjson_really_inline simdjson_result operator[](std::string_view key) & noexcept; - simdjson_really_inline simdjson_result operator[](std::string_view key) && noexcept; - simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + simdjson_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result end() noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) && noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) && noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -26871,9 +26174,9 @@ class parser { inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; inline parser(parser &&other) noexcept = default; - simdjson_really_inline parser(const parser &other) = delete; - simdjson_really_inline parser &operator=(const parser &other) = delete; - simdjson_really_inline parser &operator=(parser &&other) noexcept = default; + simdjson_inline parser(const parser &other) = delete; + simdjson_inline parser &operator=(const parser &other) = delete; + simdjson_inline parser &operator=(parser &&other) noexcept = default; /** Deallocate the JSON parser. */ inline ~parser() noexcept = default; @@ -27056,17 +26359,26 @@ class parser { simdjson_result iterate_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** The capacity of this parser (the largest document it can process). */ - simdjson_really_inline size_t capacity() const noexcept; + simdjson_inline size_t capacity() const noexcept; /** The maximum capacity of this parser (the largest document it is allowed to process). */ - simdjson_really_inline size_t max_capacity() const noexcept; - simdjson_really_inline void set_max_capacity(size_t max_capacity) noexcept; - /** The maximum depth of this parser (the most deeply nested objects and arrays it can process). */ - simdjson_really_inline size_t max_depth() const noexcept; + simdjson_inline size_t max_capacity() const noexcept; + simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; + /** + * The maximum depth of this parser (the most deeply nested objects and arrays it can process). + * This parameter is only relevant when the macro SIMDJSON_DEVELOPMENT_CHECKS is set to true. + * The document's instance current_depth() method should be used to monitor the parsing + * depth and limit it if desired. + */ + simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length * and `max_depth` depth. * + * The max_depth parameter is only relevant when the macro SIMDJSON_DEVELOPMENT_CHECKS is set to true. + * The document's instance current_depth() method should be used to monitor the parsing + * depth and limit it if desired. + * * @param capacity The new capacity. * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. * @return The error, if there is one. @@ -27082,6 +26394,27 @@ class parser { bool threaded{true}; #endif + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. + * The provided pointer is advanced to the end of the string by reference, and a string_view instance + * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least + * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer. + * + * This unescape function is a low-level function. If you want a more user-friendly approach, you should + * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string() + * instead of get_raw_json_string()). + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid as long as the bytes in dst. + * + * @param raw_json_string input + * @param dst A pointer to a buffer at least large enough to write this string as well as + * an additional SIMDJSON_PADDING bytes. + * @return A string_view pointing at the unescaped string in dst + * @error STRING_ERROR if escapes are incorrect. + */ + simdjson_inline simdjson_result unescape(raw_json_string in, uint8_t *&dst) const noexcept; private: /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; @@ -27089,7 +26422,7 @@ class parser { size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif @@ -27106,9 +26439,9 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::parser &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::parser &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; }; } // namespace simdjson @@ -27196,13 +26529,13 @@ class document_stream { * auto error = parser.iterate_many(json).get(docs); * ``` */ - simdjson_really_inline document_stream() noexcept; + simdjson_inline document_stream() noexcept; /** Move one document_stream to another. */ - simdjson_really_inline document_stream(document_stream &&other) noexcept = default; + simdjson_inline document_stream(document_stream &&other) noexcept = default; /** Move one document_stream to another. */ - simdjson_really_inline document_stream &operator=(document_stream &&other) noexcept = default; + simdjson_inline document_stream &operator=(document_stream &&other) noexcept = default; - simdjson_really_inline ~document_stream() noexcept; + simdjson_inline ~document_stream() noexcept; /** * Returns the input size in bytes. @@ -27241,11 +26574,11 @@ class document_stream { /** * Default constructor. */ - simdjson_really_inline iterator() noexcept; + simdjson_inline iterator() noexcept; /** * Get the current document (or error). */ - simdjson_really_inline simdjson_result operator*() noexcept; + simdjson_inline simdjson_result operator*() noexcept; /** * Advance to the next document (prefix). */ @@ -27254,7 +26587,7 @@ class document_stream { * Check if we're at the end yet. * @param other the end iterator to compare to. */ - simdjson_really_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator!=(const iterator &other) const noexcept; /** * @private * @@ -27270,7 +26603,7 @@ class document_stream { * may change in future versions of simdjson: we find the API somewhat * awkward and we would like to offer something friendlier. */ - simdjson_really_inline size_t current_index() const noexcept; + simdjson_inline size_t current_index() const noexcept; /** * @private @@ -27291,7 +26624,7 @@ class document_stream { * awkward and we would like to offer something friendlier. * */ - simdjson_really_inline std::string_view source() const noexcept; + simdjson_inline std::string_view source() const noexcept; /** * Returns error of the stream (if any). @@ -27299,7 +26632,7 @@ class document_stream { inline error_code error() const noexcept; private: - simdjson_really_inline iterator(document_stream *s, bool finished) noexcept; + simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ document_stream* stream; /** Whether we're finished or not. */ @@ -27313,11 +26646,11 @@ class document_stream { /** * Start iterating the documents in the stream. */ - simdjson_really_inline iterator begin() noexcept; + simdjson_inline iterator begin() noexcept; /** * The end of the stream, for iterator comparison purposes. */ - simdjson_really_inline iterator end() noexcept; + simdjson_inline iterator end() noexcept; private: @@ -27333,7 +26666,7 @@ class document_stream { * @param len is the length of the raw byte buffer in bytes * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) */ - simdjson_really_inline document_stream( + simdjson_inline document_stream( ondemand::parser &parser, const uint8_t *buf, size_t len, @@ -27437,9 +26770,9 @@ namespace simdjson { template<> struct simdjson_result : public SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base { public: - simdjson_really_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_stream &&value) noexcept; ///< @private - simdjson_really_inline simdjson_result(error_code error) noexcept; ///< @private - simdjson_really_inline simdjson_result() noexcept = default; + simdjson_inline simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_stream &&value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; }; } // namespace simdjson @@ -27449,23 +26782,27 @@ struct simdjson_result to_json_string(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document& x) noexcept; /** * Create a string-view instance out of a value instance. The string-view instance * contains JSON text that is suitable to be parsed as JSON again. The value must - * not have been accessed previously. + * not have been accessed previously. It does not + * validate the content. */ inline simdjson_result to_json_string(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value& x) noexcept; /** * Create a string-view instance out of an object instance. The string-view instance - * contains JSON text that is suitable to be parsed as JSON again. + * contains JSON text that is suitable to be parsed as JSON again. It does not + * validate the content. */ inline simdjson_result to_json_string(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object& x) noexcept; /** * Create a string-view instance out of an array instance. The string-view instance - * contains JSON text that is suitable to be parsed as JSON again. + * contains JSON text that is suitable to be parsed as JSON again. It does not + * validate the content. */ inline simdjson_result to_json_string(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array& x) noexcept; inline simdjson_result to_json_string(simdjson_result x); @@ -27484,7 +26821,8 @@ inline simdjson_result to_json_string(simdjson_result x); #endif /** - * Print JSON to an output stream. + * Print JSON to an output stream. It does not + * validate the content. * * @param out The output stream. * @param value The array. @@ -27506,7 +26845,8 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_BUILTIN_IM inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); #endif /** - * Print JSON to an output stream. + * Print JSON to an output stream. It does not + * validate the content. * * @param out The output stream. * @param value The array. @@ -27521,7 +26861,8 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_BUILTIN_IM inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result&& x); #endif /** - * Print JSON to an output stream. + * Print JSON to an output stream. It does not + * validate the content. * * @param out The output stream. * @param value The object. @@ -27545,7 +26886,7 @@ namespace SIMDJSON_BUILTIN_IMPLEMENTATION { // template -simdjson_really_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { +simdjson_inline void implementation_simdjson_result_base::tie(T &value, error_code &error) && noexcept { error = this->second; if (!error) { value = std::forward>(*this).first; @@ -27553,66 +26894,66 @@ simdjson_really_inline void implementation_simdjson_result_base::tie(T &value } template -simdjson_warn_unused simdjson_really_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { +simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T &value) && noexcept { error_code error; std::forward>(*this).tie(value, error); return error; } template -simdjson_really_inline error_code implementation_simdjson_result_base::error() const noexcept { +simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { return this->second; } #if SIMDJSON_EXCEPTIONS template -simdjson_really_inline T& implementation_simdjson_result_base::value() & noexcept(false) { +simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return this->first; } template -simdjson_really_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { +simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { return std::forward>(*this).take_value(); } template -simdjson_really_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { +simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { if (error()) { throw simdjson_error(error()); } return std::forward(this->first); } template -simdjson_really_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { +simdjson_inline implementation_simdjson_result_base::operator T&&() && noexcept(false) { return std::forward>(*this).take_value(); } #endif // SIMDJSON_EXCEPTIONS template -simdjson_really_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { +simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { return this->first; } template -simdjson_really_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { +simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { return this->first; } template -simdjson_really_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { +simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { return std::forward(this->first); } template -simdjson_really_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept +simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value, error_code error) noexcept : first{std::forward(value)}, second{error} {} template -simdjson_really_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept +simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept : implementation_simdjson_result_base(T{}, error) {} template -simdjson_really_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept +simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T &&value) noexcept : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} } // namespace SIMDJSON_BUILTIN_IMPLEMENTATION @@ -27654,48 +26995,48 @@ inline std::ostream& operator<<(std::ostream& out, simdjson_result &t -simdjson_really_inline number_type number::get_number_type() const noexcept { +simdjson_inline number_type number::get_number_type() const noexcept { return type; } -simdjson_really_inline bool number::is_uint64() const noexcept { +simdjson_inline bool number::is_uint64() const noexcept { return get_number_type() == number_type::unsigned_integer; } -simdjson_really_inline uint64_t number::get_uint64() const noexcept { +simdjson_inline uint64_t number::get_uint64() const noexcept { return payload.unsigned_integer; } -simdjson_really_inline number::operator uint64_t() const noexcept { +simdjson_inline number::operator uint64_t() const noexcept { return get_uint64(); } -simdjson_really_inline bool number::is_int64() const noexcept { +simdjson_inline bool number::is_int64() const noexcept { return get_number_type() == number_type::signed_integer; } -simdjson_really_inline int64_t number::get_int64() const noexcept { +simdjson_inline int64_t number::get_int64() const noexcept { return payload.signed_integer; } -simdjson_really_inline number::operator int64_t() const noexcept { +simdjson_inline number::operator int64_t() const noexcept { return get_int64(); } -simdjson_really_inline bool number::is_double() const noexcept { +simdjson_inline bool number::is_double() const noexcept { return get_number_type() == number_type::floating_point_number; } -simdjson_really_inline double number::get_double() const noexcept { +simdjson_inline double number::get_double() const noexcept { return payload.floating_point_number; } -simdjson_really_inline number::operator double() const noexcept { +simdjson_inline number::operator double() const noexcept { return get_double(); } -simdjson_really_inline double number::as_double() const noexcept { +simdjson_inline double number::as_double() const noexcept { if(is_double()) { return payload.floating_point_number; } @@ -27705,22 +27046,22 @@ simdjson_really_inline double number::as_double() const noexcept { return double(payload.unsigned_integer); } -simdjson_really_inline void number::append_s64(int64_t value) noexcept { +simdjson_inline void number::append_s64(int64_t value) noexcept { payload.signed_integer = value; type = number_type::signed_integer; } -simdjson_really_inline void number::append_u64(uint64_t value) noexcept { +simdjson_inline void number::append_u64(uint64_t value) noexcept { payload.unsigned_integer = value; type = number_type::unsigned_integer; } -simdjson_really_inline void number::append_double(double value) noexcept { +simdjson_inline void number::append_double(double value) noexcept { payload.floating_point_number = value; type = number_type::floating_point_number; } -simdjson_really_inline void number::skip_double() noexcept { +simdjson_inline void number::skip_double() noexcept { type = number_type::floating_point_number; } @@ -27730,9 +27071,9 @@ simdjson_really_inline void number::skip_double() noexcept { namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_type &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_type &&value) noexcept : implementation_simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : implementation_simdjson_result_base(error) {} } // namespace simdjson @@ -27892,7 +27233,7 @@ inline void log_line(const json_iterator &iter, token_position index, depth_t de printf(" "); } // printf("| %5u ", *(index+1)); - printf("| %5u ", depth); + printf("| %5i ", depth); printf("| %.*s ", int(detail.size()), detail.data()); printf("|\n"); fflush(stdout); @@ -27910,18 +27251,12 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline raw_json_string::raw_json_string(const uint8_t * _buf) noexcept : buf{_buf} {} +simdjson_inline raw_json_string::raw_json_string(const uint8_t * _buf) noexcept : buf{_buf} {} -simdjson_really_inline const char * raw_json_string::raw() const noexcept { return reinterpret_cast(buf); } -simdjson_really_inline simdjson_warn_unused simdjson_result raw_json_string::unescape(uint8_t *&dst) const noexcept { - uint8_t *end = stringparsing::parse_string(buf, dst); - if (!end) { return STRING_ERROR; } - std::string_view result(reinterpret_cast(dst), end-dst); - dst = end; - return result; -} +simdjson_inline const char * raw_json_string::raw() const noexcept { return reinterpret_cast(buf); } -simdjson_really_inline bool raw_json_string::is_free_from_unescaped_quote(std::string_view target) noexcept { + +simdjson_inline bool raw_json_string::is_free_from_unescaped_quote(std::string_view target) noexcept { size_t pos{0}; // if the content has no escape character, just scan through it quickly! for(;pos < target.size() && target[pos] != '\\';pos++) {} @@ -27939,7 +27274,7 @@ simdjson_really_inline bool raw_json_string::is_free_from_unescaped_quote(std::s return true; } -simdjson_really_inline bool raw_json_string::is_free_from_unescaped_quote(const char* target) noexcept { +simdjson_inline bool raw_json_string::is_free_from_unescaped_quote(const char* target) noexcept { size_t pos{0}; // if the content has no escape character, just scan through it quickly! for(;target[pos] && target[pos] != '\\';pos++) {} @@ -27958,12 +27293,12 @@ simdjson_really_inline bool raw_json_string::is_free_from_unescaped_quote(const } -simdjson_really_inline bool raw_json_string::unsafe_is_equal(size_t length, std::string_view target) const noexcept { +simdjson_inline bool raw_json_string::unsafe_is_equal(size_t length, std::string_view target) const noexcept { // If we are going to call memcmp, then we must know something about the length of the raw_json_string. return (length >= target.size()) && (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); } -simdjson_really_inline bool raw_json_string::unsafe_is_equal(std::string_view target) const noexcept { +simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) const noexcept { // Assumptions: does not contain unescaped quote characters, and // the raw content is quote terminated within a valid JSON string. if(target.size() <= SIMDJSON_PADDING) { @@ -27978,7 +27313,7 @@ simdjson_really_inline bool raw_json_string::unsafe_is_equal(std::string_view ta return true; } -simdjson_really_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { +simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { const char * r{raw()}; size_t pos{0}; bool escaping{false}; @@ -28002,7 +27337,7 @@ simdjson_really_inline bool raw_json_string::is_equal(std::string_view target) c } -simdjson_really_inline bool raw_json_string::unsafe_is_equal(const char * target) const noexcept { +simdjson_inline bool raw_json_string::unsafe_is_equal(const char * target) const noexcept { // Assumptions: 'target' does not contain unescaped quote characters, is null terminated and // the raw content is quote terminated within a valid JSON string. const char * r{raw()}; @@ -28014,7 +27349,7 @@ simdjson_really_inline bool raw_json_string::unsafe_is_equal(const char * target return true; } -simdjson_really_inline bool raw_json_string::is_equal(const char* target) const noexcept { +simdjson_inline bool raw_json_string::is_equal(const char* target) const noexcept { // Assumptions: does not contain unescaped quote characters, and // the raw content is quote terminated within a valid JSON string. const char * r{raw()}; @@ -28039,29 +27374,29 @@ simdjson_really_inline bool raw_json_string::is_equal(const char* target) const return true; } -simdjson_unused simdjson_really_inline bool operator==(const raw_json_string &a, std::string_view c) noexcept { +simdjson_unused simdjson_inline bool operator==(const raw_json_string &a, std::string_view c) noexcept { return a.unsafe_is_equal(c); } -simdjson_unused simdjson_really_inline bool operator==(std::string_view c, const raw_json_string &a) noexcept { +simdjson_unused simdjson_inline bool operator==(std::string_view c, const raw_json_string &a) noexcept { return a == c; } -simdjson_unused simdjson_really_inline bool operator!=(const raw_json_string &a, std::string_view c) noexcept { +simdjson_unused simdjson_inline bool operator!=(const raw_json_string &a, std::string_view c) noexcept { return !(a == c); } -simdjson_unused simdjson_really_inline bool operator!=(std::string_view c, const raw_json_string &a) noexcept { +simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_json_string &a) noexcept { return !(a == c); } -simdjson_really_inline simdjson_warn_unused simdjson_result raw_json_string::unescape(json_iterator &iter) const noexcept { - return unescape(iter.string_buf_loc()); +simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape(json_iterator &iter) const noexcept { + return iter.unescape(*this); } -simdjson_unused simdjson_really_inline std::ostream &operator<<(std::ostream &out, const raw_json_string &str) noexcept { +simdjson_unused simdjson_inline std::ostream &operator<<(std::ostream &out, const raw_json_string &str) noexcept { bool in_escape = false; const char *s = str.raw(); while (true) { @@ -28081,20 +27416,16 @@ simdjson_unused simdjson_really_inline std::ostream &operator<<(std::ostream &ou namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string &&value) noexcept : implementation_simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : implementation_simdjson_result_base(error) {} -simdjson_really_inline simdjson_result simdjson_result::raw() const noexcept { +simdjson_inline simdjson_result simdjson_result::raw() const noexcept { if (error()) { return error(); } return first.raw(); } -simdjson_really_inline simdjson_warn_unused simdjson_result simdjson_result::unescape(uint8_t *&dst) const noexcept { - if (error()) { return error(); } - return first.unescape(dst); -} -simdjson_really_inline simdjson_warn_unused simdjson_result simdjson_result::unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept { +simdjson_inline simdjson_warn_unused simdjson_result simdjson_result::unescape(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept { if (error()) { return error(); } return first.unescape(iter); } @@ -28106,65 +27437,65 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline token_iterator::token_iterator( +simdjson_inline token_iterator::token_iterator( const uint8_t *_buf, token_position position ) noexcept : buf{_buf}, _position{position} { } -simdjson_really_inline uint32_t token_iterator::current_offset() const noexcept { +simdjson_inline uint32_t token_iterator::current_offset() const noexcept { return *(_position); } -simdjson_really_inline const uint8_t *token_iterator::return_current_and_advance() noexcept { +simdjson_inline const uint8_t *token_iterator::return_current_and_advance() noexcept { return &buf[*(_position++)]; } -simdjson_really_inline const uint8_t *token_iterator::peek(token_position position) const noexcept { +simdjson_inline const uint8_t *token_iterator::peek(token_position position) const noexcept { return &buf[*position]; } -simdjson_really_inline uint32_t token_iterator::peek_index(token_position position) const noexcept { +simdjson_inline uint32_t token_iterator::peek_index(token_position position) const noexcept { return *position; } -simdjson_really_inline uint32_t token_iterator::peek_length(token_position position) const noexcept { +simdjson_inline uint32_t token_iterator::peek_length(token_position position) const noexcept { return *(position+1) - *position; } -simdjson_really_inline const uint8_t *token_iterator::peek(int32_t delta) const noexcept { +simdjson_inline const uint8_t *token_iterator::peek(int32_t delta) const noexcept { return &buf[*(_position+delta)]; } -simdjson_really_inline uint32_t token_iterator::peek_index(int32_t delta) const noexcept { +simdjson_inline uint32_t token_iterator::peek_index(int32_t delta) const noexcept { return *(_position+delta); } -simdjson_really_inline uint32_t token_iterator::peek_length(int32_t delta) const noexcept { +simdjson_inline uint32_t token_iterator::peek_length(int32_t delta) const noexcept { return *(_position+delta+1) - *(_position+delta); } -simdjson_really_inline token_position token_iterator::position() const noexcept { +simdjson_inline token_position token_iterator::position() const noexcept { return _position; } -simdjson_really_inline void token_iterator::set_position(token_position target_position) noexcept { +simdjson_inline void token_iterator::set_position(token_position target_position) noexcept { _position = target_position; } -simdjson_really_inline bool token_iterator::operator==(const token_iterator &other) const noexcept { +simdjson_inline bool token_iterator::operator==(const token_iterator &other) const noexcept { return _position == other._position; } -simdjson_really_inline bool token_iterator::operator!=(const token_iterator &other) const noexcept { +simdjson_inline bool token_iterator::operator!=(const token_iterator &other) const noexcept { return _position != other._position; } -simdjson_really_inline bool token_iterator::operator>(const token_iterator &other) const noexcept { +simdjson_inline bool token_iterator::operator>(const token_iterator &other) const noexcept { return _position > other._position; } -simdjson_really_inline bool token_iterator::operator>=(const token_iterator &other) const noexcept { +simdjson_inline bool token_iterator::operator>=(const token_iterator &other) const noexcept { return _position >= other._position; } -simdjson_really_inline bool token_iterator::operator<(const token_iterator &other) const noexcept { +simdjson_inline bool token_iterator::operator<(const token_iterator &other) const noexcept { return _position < other._position; } -simdjson_really_inline bool token_iterator::operator<=(const token_iterator &other) const noexcept { +simdjson_inline bool token_iterator::operator<=(const token_iterator &other) const noexcept { return _position <= other._position; } @@ -28174,9 +27505,9 @@ simdjson_really_inline bool token_iterator::operator<=(const token_iterator &oth namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::token_iterator &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::token_iterator &&value) noexcept : implementation_simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : implementation_simdjson_result_base(error) {} } // namespace simdjson @@ -28186,7 +27517,7 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline json_iterator::json_iterator(json_iterator &&other) noexcept +simdjson_inline json_iterator::json_iterator(json_iterator &&other) noexcept : token(std::forward(other.token)), parser{other.parser}, _string_buf_loc{other._string_buf_loc}, @@ -28197,7 +27528,7 @@ simdjson_really_inline json_iterator::json_iterator(json_iterator &&other) noexc { other.parser = nullptr; } -simdjson_really_inline json_iterator &json_iterator::operator=(json_iterator &&other) noexcept { +simdjson_inline json_iterator &json_iterator::operator=(json_iterator &&other) noexcept { token = other.token; parser = other.parser; _string_buf_loc = other._string_buf_loc; @@ -28209,7 +27540,7 @@ simdjson_really_inline json_iterator &json_iterator::operator=(json_iterator &&o return *this; } -simdjson_really_inline json_iterator::json_iterator(const uint8_t *buf, ondemand::parser *_parser) noexcept +simdjson_inline json_iterator::json_iterator(const uint8_t *buf, ondemand::parser *_parser) noexcept : token(buf, &_parser->implementation->structural_indexes[0]), parser{_parser}, _string_buf_loc{parser->string_buf.get()}, @@ -28231,12 +27562,33 @@ inline void json_iterator::rewind() noexcept { _depth = 1; } +inline bool json_iterator::balanced() const noexcept { + token_iterator ti(token); + int32_t count{0}; + ti.set_position( root_position() ); + while(ti.peek() <= peek_last()) { + switch (*ti.return_current_and_advance()) + { + case '[': case '{': + count++; + break; + case ']': case '}': + count--; + break; + default: + break; + } + } + return count == 0; +} + + // GCC 7 warns when the first line of this function is inlined away into oblivion due to the caller // relating depth and parent_depth, which is a desired effect. The warning does not show up if the // skip_child() function is not marked inline). SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING -simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child(depth_t parent_depth) noexcept { +simdjson_warn_unused simdjson_inline error_code json_iterator::skip_child(depth_t parent_depth) noexcept { if (depth() <= parent_depth) { return SUCCESS; } switch (*return_current_and_advance()) { // TODO consider whether matching braces is a requirement: if non-matching braces indicates @@ -28313,23 +27665,27 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child SIMDJSON_POP_DISABLE_WARNINGS -simdjson_really_inline bool json_iterator::at_root() const noexcept { +simdjson_inline bool json_iterator::at_root() const noexcept { return position() == root_position(); } -simdjson_really_inline bool json_iterator::streaming() const noexcept { +simdjson_inline bool json_iterator::is_single_token() const noexcept { + return parser->implementation->n_structural_indexes == 1; +} + +simdjson_inline bool json_iterator::streaming() const noexcept { return _streaming; } -simdjson_really_inline token_position json_iterator::root_position() const noexcept { +simdjson_inline token_position json_iterator::root_position() const noexcept { return _root; } -simdjson_really_inline void json_iterator::assert_at_document_depth() const noexcept { +simdjson_inline void json_iterator::assert_at_document_depth() const noexcept { SIMDJSON_ASSUME( _depth == 1 ); } -simdjson_really_inline void json_iterator::assert_at_root() const noexcept { +simdjson_inline void json_iterator::assert_at_root() const noexcept { SIMDJSON_ASSUME( _depth == 1 ); #ifndef SIMDJSON_CLANG_VISUAL_STUDIO // Under Visual Studio, the next SIMDJSON_ASSUME fails with: the argument @@ -28338,21 +27694,21 @@ simdjson_really_inline void json_iterator::assert_at_root() const noexcept { #endif } -simdjson_really_inline void json_iterator::assert_more_tokens(uint32_t required_tokens) const noexcept { +simdjson_inline void json_iterator::assert_more_tokens(uint32_t required_tokens) const noexcept { assert_valid_position(token._position + required_tokens - 1); } -simdjson_really_inline void json_iterator::assert_valid_position(token_position position) const noexcept { +simdjson_inline void json_iterator::assert_valid_position(token_position position) const noexcept { #ifndef SIMDJSON_CLANG_VISUAL_STUDIO SIMDJSON_ASSUME( position >= &parser->implementation->structural_indexes[0] ); SIMDJSON_ASSUME( position < &parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] ); #endif } -simdjson_really_inline bool json_iterator::at_end() const noexcept { +simdjson_inline bool json_iterator::at_end() const noexcept { return position() == end_position(); } -simdjson_really_inline token_position json_iterator::end_position() const noexcept { +simdjson_inline token_position json_iterator::end_position() const noexcept { uint32_t n_structural_indexes{parser->implementation->n_structural_indexes}; return &parser->implementation->structural_indexes[n_structural_indexes]; } @@ -28381,42 +27737,42 @@ inline simdjson_result json_iterator::current_location() noexcept return reinterpret_cast(token.peek()); } -simdjson_really_inline bool json_iterator::is_alive() const noexcept { +simdjson_inline bool json_iterator::is_alive() const noexcept { return parser; } -simdjson_really_inline void json_iterator::abandon() noexcept { +simdjson_inline void json_iterator::abandon() noexcept { parser = nullptr; _depth = 0; } -simdjson_really_inline const uint8_t *json_iterator::return_current_and_advance() noexcept { +simdjson_inline const uint8_t *json_iterator::return_current_and_advance() noexcept { #if SIMDJSON_CHECK_EOF assert_more_tokens(); #endif // SIMDJSON_CHECK_EOF return token.return_current_and_advance(); } -simdjson_really_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept { +simdjson_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept { // deliberately done without safety guard: return token.peek(0); } -simdjson_really_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept { +simdjson_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept { #if SIMDJSON_CHECK_EOF assert_more_tokens(delta+1); #endif // SIMDJSON_CHECK_EOF return token.peek(delta); } -simdjson_really_inline uint32_t json_iterator::peek_length(int32_t delta) const noexcept { +simdjson_inline uint32_t json_iterator::peek_length(int32_t delta) const noexcept { #if SIMDJSON_CHECK_EOF assert_more_tokens(delta+1); #endif // #if SIMDJSON_CHECK_EOF return token.peek_length(delta); } -simdjson_really_inline const uint8_t *json_iterator::peek(token_position position) const noexcept { +simdjson_inline const uint8_t *json_iterator::peek(token_position position) const noexcept { // todo: currently we require end-of-string buffering, but the following // assert_valid_position should be turned on if/when we lift that condition. // assert_valid_position(position); @@ -28425,14 +27781,14 @@ simdjson_really_inline const uint8_t *json_iterator::peek(token_position positio return token.peek(position); } -simdjson_really_inline uint32_t json_iterator::peek_length(token_position position) const noexcept { +simdjson_inline uint32_t json_iterator::peek_length(token_position position) const noexcept { #if SIMDJSON_CHECK_EOF assert_valid_position(position); #endif // SIMDJSON_CHECK_EOF return token.peek_length(position); } -simdjson_really_inline token_position json_iterator::last_position() const noexcept { +simdjson_inline token_position json_iterator::last_position() const noexcept { // The following line fails under some compilers... // SIMDJSON_ASSUME(parser->implementation->n_structural_indexes > 0); // since it has side-effects. @@ -28440,46 +27796,51 @@ simdjson_really_inline token_position json_iterator::last_position() const noexc SIMDJSON_ASSUME(n_structural_indexes > 0); return &parser->implementation->structural_indexes[n_structural_indexes - 1]; } -simdjson_really_inline const uint8_t *json_iterator::peek_last() const noexcept { +simdjson_inline const uint8_t *json_iterator::peek_last() const noexcept { return token.peek(last_position()); } -simdjson_really_inline void json_iterator::ascend_to(depth_t parent_depth) noexcept { +simdjson_inline void json_iterator::ascend_to(depth_t parent_depth) noexcept { SIMDJSON_ASSUME(parent_depth >= 0 && parent_depth < INT32_MAX - 1); SIMDJSON_ASSUME(_depth == parent_depth + 1); _depth = parent_depth; } -simdjson_really_inline void json_iterator::descend_to(depth_t child_depth) noexcept { +simdjson_inline void json_iterator::descend_to(depth_t child_depth) noexcept { SIMDJSON_ASSUME(child_depth >= 1 && child_depth < INT32_MAX); SIMDJSON_ASSUME(_depth == child_depth - 1); _depth = child_depth; } -simdjson_really_inline depth_t json_iterator::depth() const noexcept { +simdjson_inline depth_t json_iterator::depth() const noexcept { return _depth; } -simdjson_really_inline uint8_t *&json_iterator::string_buf_loc() noexcept { +simdjson_inline uint8_t *&json_iterator::string_buf_loc() noexcept { return _string_buf_loc; } -simdjson_really_inline error_code json_iterator::report_error(error_code _error, const char *message) noexcept { +simdjson_inline error_code json_iterator::report_error(error_code _error, const char *message) noexcept { SIMDJSON_ASSUME(_error != SUCCESS && _error != UNINITIALIZED && _error != INCORRECT_TYPE && _error != NO_SUCH_FIELD); logger::log_error(*this, message); error = _error; return error; } -simdjson_really_inline token_position json_iterator::position() const noexcept { +simdjson_inline token_position json_iterator::position() const noexcept { return token.position(); } -simdjson_really_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { +simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in) noexcept { + return parser->unescape(in, _string_buf_loc); +} + +simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { SIMDJSON_ASSUME(child_depth >= 1 && child_depth < INT32_MAX); SIMDJSON_ASSUME(_depth == child_depth - 1); -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS #ifndef SIMDJSON_CLANG_VISUAL_STUDIO + SIMDJSON_ASSUME(size_t(child_depth) < parser->max_depth()); SIMDJSON_ASSUME(position >= parser->start_positions[child_depth]); #endif #endif @@ -28487,27 +27848,29 @@ simdjson_really_inline void json_iterator::reenter_child(token_position position _depth = child_depth; } -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS -simdjson_really_inline token_position json_iterator::start_position(depth_t depth) const noexcept { - return parser->start_positions[depth]; +simdjson_inline token_position json_iterator::start_position(depth_t depth) const noexcept { + SIMDJSON_ASSUME(size_t(depth) < parser->max_depth()); + return size_t(depth) < parser->max_depth() ? parser->start_positions[depth] : 0; } -simdjson_really_inline void json_iterator::set_start_position(depth_t depth, token_position position) noexcept { - parser->start_positions[depth] = position; +simdjson_inline void json_iterator::set_start_position(depth_t depth, token_position position) noexcept { + SIMDJSON_ASSUME(size_t(depth) < parser->max_depth()); + if(size_t(depth) < parser->max_depth()) { parser->start_positions[depth] = position; } } #endif -simdjson_really_inline error_code json_iterator::optional_error(error_code _error, const char *message) noexcept { +simdjson_inline error_code json_iterator::optional_error(error_code _error, const char *message) noexcept { SIMDJSON_ASSUME(_error == INCORRECT_TYPE || _error == NO_SUCH_FIELD); logger::log_error(*this, message); return _error; } template -simdjson_warn_unused simdjson_really_inline bool json_iterator::copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept { +simdjson_warn_unused simdjson_inline bool json_iterator::copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept { // Let us guard against silly cases: if((N < max_len) || (N == 0)) { return false; } // Truncate whitespace to fit the buffer. @@ -28528,9 +27891,9 @@ simdjson_warn_unused simdjson_really_inline bool json_iterator::copy_to_buffer(c namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::json_iterator &&value) noexcept : implementation_simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : implementation_simdjson_result_base(error) {} } // namespace simdjson @@ -28540,7 +27903,7 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline value_iterator::value_iterator( +simdjson_inline value_iterator::value_iterator( json_iterator *json_iter, depth_t depth, token_position start_position @@ -28548,19 +27911,19 @@ simdjson_really_inline value_iterator::value_iterator( { } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::start_object() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_object() noexcept { SIMDJSON_TRY( start_container('{', "Not an object", "object") ); return started_object(); } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::start_root_object() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_root_object() noexcept { SIMDJSON_TRY( start_container('{', "Not an object", "object") ); return started_root_object(); } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::started_object() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_object() noexcept { assert_at_container_start(); -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS _json_iter->set_start_position(_depth, start_position()); #endif if (*_json_iter->peek() == '}') { @@ -28572,19 +27935,32 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator return true; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::started_root_object() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_root_object() noexcept { // When in streaming mode, we cannot expect peek_last() to be the last structural element of the // current document. It only works in the normal mode where we have indexed a single document. // Note that adding a check for 'streaming' is not expensive since we only have at most // one root element. - if (! _json_iter->streaming() && (*_json_iter->peek_last() != '}')) { - _json_iter->abandon(); - return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing } at end"); + if ( ! _json_iter->streaming() ) { + if (*_json_iter->peek_last() != '}') { + _json_iter->abandon(); + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing } at end"); + } + // If the last character is } *and* the first gibberish character is also '}' + // then on-demand could accidentally go over. So we need additional checks. + // https://github.com/simdjson/simdjson/issues/1834 + // Checking that the document is balanced requires a full scan which is potentially + // expensive, but it only happens in edge cases where the first padding character is + // a closing bracket. + if ((*_json_iter->peek(_json_iter->end_position()) == '}') && (!_json_iter->balanced())) { + _json_iter->abandon(); + // The exact error would require more work. It will typically be an unclosed object. + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "the document is unbalanced"); + } } return started_object(); } -simdjson_warn_unused simdjson_really_inline error_code value_iterator::end_container() noexcept { +simdjson_warn_unused simdjson_inline error_code value_iterator::end_container() noexcept { #if SIMDJSON_CHECK_EOF if (depth() > 1 && at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing parent ] or }"); } // if (depth() <= 1 && !at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing [ or { at start"); } @@ -28593,7 +27969,7 @@ simdjson_warn_unused simdjson_really_inline error_code value_iterator::end_conta return SUCCESS; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::has_next_field() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::has_next_field() noexcept { assert_at_next(); // It's illegal to call this unless there are more tokens: anything that ends in } or ] is @@ -28610,7 +27986,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator } } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::find_field_raw(const std::string_view key) noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::find_field_raw(const std::string_view key) noexcept { error_code error; bool has_value; // @@ -28636,7 +28012,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator // ``` // } else if (!is_open()) { -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. @@ -28661,7 +28037,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator } else { if ((error = skip_child() )) { abandon(); return error; } if ((error = has_next_field().get(has_value) )) { abandon(); return error; } -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS if (_json_iter->start_position(_depth) != start_position()) { return OUT_OF_ORDER_ITERATION; } #endif } @@ -28705,7 +28081,9 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator return false; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::find_field_unordered_raw(const std::string_view key) noexcept { +SIMDJSON_PUSH_DISABLE_WARNINGS +SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::find_field_unordered_raw(const std::string_view key) noexcept { /** * When find_field_unordered_raw is called, we can either be pointing at the * first key, pointing outside (at the closing brace) or if a key was matched @@ -28746,7 +28124,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator // } else if (!is_open()) { -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. @@ -28775,7 +28153,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator if ((error = skip_child() )) { abandon(); return error; } search_start = _json_iter->position(); if ((error = has_next_field().get(has_value) )) { abandon(); return error; } -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS if (_json_iter->start_position(_depth) != start_position()) { return OUT_OF_ORDER_ITERATION; } #endif } @@ -28892,8 +28270,9 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator // never reach this point. return false; } +SIMDJSON_POP_DISABLE_WARNINGS -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::field_key() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::field_key() noexcept { assert_at_next(); const uint8_t *key = _json_iter->return_current_and_advance(); @@ -28901,7 +28280,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result val return raw_json_string(key); } -simdjson_warn_unused simdjson_really_inline error_code value_iterator::field_value() noexcept { +simdjson_warn_unused simdjson_inline error_code value_iterator::field_value() noexcept { assert_at_next(); if (*_json_iter->return_current_and_advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); } @@ -28909,12 +28288,12 @@ simdjson_warn_unused simdjson_really_inline error_code value_iterator::field_val return SUCCESS; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::start_array() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_array() noexcept { SIMDJSON_TRY( start_container('[', "Not an array", "array") ); return started_array(); } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::start_root_array() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_root_array() noexcept { SIMDJSON_TRY( start_container('[', "Not an array", "array") ); return started_root_array(); } @@ -28926,7 +28305,7 @@ inline std::string value_iterator::to_string() const noexcept { return answer; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::started_array() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_array() noexcept { assert_at_container_start(); if (*_json_iter->peek() == ']') { logger::log_value(*_json_iter, "empty array"); @@ -28935,25 +28314,38 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator return false; } _json_iter->descend_to(depth()+1); -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS _json_iter->set_start_position(_depth, start_position()); #endif return true; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::started_root_array() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_root_array() noexcept { // When in streaming mode, we cannot expect peek_last() to be the last structural element of the // current document. It only works in the normal mode where we have indexed a single document. // Note that adding a check for 'streaming' is not expensive since we only have at most // one root element. - if ( ! _json_iter->streaming() && (*_json_iter->peek_last() != ']')) { - _json_iter->abandon(); - return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing ] at end"); + if ( ! _json_iter->streaming() ) { + if (*_json_iter->peek_last() != ']') { + _json_iter->abandon(); + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing ] at end"); + } + // If the last character is ] *and* the first gibberish character is also ']' + // then on-demand could accidentally go over. So we need additional checks. + // https://github.com/simdjson/simdjson/issues/1834 + // Checking that the document is balanced requires a full scan which is potentially + // expensive, but it only happens in edge cases where the first padding character is + // a closing bracket. + if ((*_json_iter->peek(_json_iter->end_position()) == ']') && (!_json_iter->balanced())) { + _json_iter->abandon(); + // The exact error would require more work. It will typically be an unclosed array. + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "the document is unbalanced"); + } } return started_array(); } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::has_next_element() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::has_next_element() noexcept { assert_at_next(); logger::log_event(*this, "has_next_element"); @@ -28970,96 +28362,106 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator } } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::parse_bool(const uint8_t *json) const noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::parse_bool(const uint8_t *json) const noexcept { auto not_true = atomparsing::str4ncmp(json, "true"); auto not_false = atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e'); bool error = (not_true && not_false) || jsoncharutils::is_not_structural_or_whitespace(json[not_true ? 5 : 4]); if (error) { return incorrect_type_error("Not a boolean"); } return simdjson_result(!not_true); } -simdjson_really_inline bool value_iterator::parse_null(const uint8_t *json) const noexcept { - return !atomparsing::str4ncmp(json, "null") && jsoncharutils::is_structural_or_whitespace(json[4]); +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::parse_null(const uint8_t *json) const noexcept { + bool is_null_string = !atomparsing::str4ncmp(json, "null") && jsoncharutils::is_structural_or_whitespace(json[4]); + // if we start with 'n', we must be a null + if(!is_null_string && json[0]=='n') { return incorrect_type_error("Not a null but starts with n"); } + return is_null_string; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_string() noexcept { - return get_raw_json_string().unescape(_json_iter->string_buf_loc()); +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_string() noexcept { + return get_raw_json_string().unescape(json_iter()); } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_raw_json_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_raw_json_string() noexcept { auto json = peek_scalar("string"); if (*json != '"') { return incorrect_type_error("Not a string"); } advance_scalar("string"); return raw_json_string(json+1); } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_uint64() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_uint64() noexcept { auto result = numberparsing::parse_unsigned(peek_non_root_scalar("uint64")); if(result.error() == SUCCESS) { advance_non_root_scalar("uint64"); } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_uint64_in_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_uint64_in_string() noexcept { auto result = numberparsing::parse_unsigned_in_string(peek_non_root_scalar("uint64")); if(result.error() == SUCCESS) { advance_non_root_scalar("uint64"); } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_int64() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_int64() noexcept { auto result = numberparsing::parse_integer(peek_non_root_scalar("int64")); if(result.error() == SUCCESS) { advance_non_root_scalar("int64"); } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_int64_in_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_int64_in_string() noexcept { auto result = numberparsing::parse_integer_in_string(peek_non_root_scalar("int64")); if(result.error() == SUCCESS) { advance_non_root_scalar("int64"); } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_double() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_double() noexcept { auto result = numberparsing::parse_double(peek_non_root_scalar("double")); if(result.error() == SUCCESS) { advance_non_root_scalar("double"); } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_double_in_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_double_in_string() noexcept { auto result = numberparsing::parse_double_in_string(peek_non_root_scalar("double")); if(result.error() == SUCCESS) { advance_non_root_scalar("double"); } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_bool() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_bool() noexcept { auto result = parse_bool(peek_non_root_scalar("bool")); if(result.error() == SUCCESS) { advance_non_root_scalar("bool"); } return result; } -simdjson_really_inline bool value_iterator::is_null() noexcept { - auto result = parse_null(peek_non_root_scalar("null")); - if(result) { advance_non_root_scalar("null"); } - return result; +simdjson_inline simdjson_result value_iterator::is_null() noexcept { + bool is_null_value; + SIMDJSON_TRY(parse_null(peek_non_root_scalar("null")).get(is_null_value)); + if(is_null_value) { advance_non_root_scalar("null"); } + return is_null_value; } -simdjson_really_inline bool value_iterator::is_negative() noexcept { +simdjson_inline bool value_iterator::is_negative() noexcept { return numberparsing::is_negative(peek_non_root_scalar("numbersign")); } -simdjson_really_inline bool value_iterator::is_root_negative() noexcept { +simdjson_inline bool value_iterator::is_root_negative() noexcept { return numberparsing::is_negative(peek_root_scalar("numbersign")); } -simdjson_really_inline simdjson_result value_iterator::is_integer() noexcept { +simdjson_inline simdjson_result value_iterator::is_integer() noexcept { return numberparsing::is_integer(peek_non_root_scalar("integer")); } -simdjson_really_inline simdjson_result value_iterator::get_number_type() noexcept { +simdjson_inline simdjson_result value_iterator::get_number_type() noexcept { return numberparsing::get_number_type(peek_non_root_scalar("integer")); } -simdjson_really_inline simdjson_result value_iterator::get_number() noexcept { +simdjson_inline simdjson_result value_iterator::get_number() noexcept { number num; error_code error = numberparsing::parse_number(peek_non_root_scalar("number"), num); if(error) { return error; } return num; } -simdjson_really_inline simdjson_result value_iterator::is_root_integer() noexcept { +simdjson_inline simdjson_result value_iterator::is_root_integer() noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("is_root_integer"); uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { return false; // if there are more than 20 characters, it cannot be represented as an integer. } - return numberparsing::is_integer(tmpbuf); + auto answer = numberparsing::is_integer(tmpbuf); + // If the parsing was a success, we must still check that it is + // a single scalar. Note that we parse first because of cases like '[]' where + // getting TRAILING_CONTENT is wrong. + if((answer.error() == SUCCESS) && (!_json_iter->is_single_token())) { return TRAILING_CONTENT; } + return answer; } -simdjson_really_inline simdjson_result value_iterator::get_root_number_type() noexcept { +simdjson_inline simdjson_result value_iterator::get_root_number_type() noexcept { + if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } auto max_len = peek_start_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, @@ -29070,9 +28472,14 @@ simdjson_really_inline simdjson_resultis_single_token())) { return TRAILING_CONTENT; } + return answer; } -simdjson_really_inline simdjson_result value_iterator::get_root_number() noexcept { +simdjson_inline simdjson_result value_iterator::get_root_number() noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, @@ -29086,17 +28493,18 @@ simdjson_really_inline simdjson_result value_iterator::get_root_number() number num; error_code error = numberparsing::parse_number(tmpbuf, num); if(error) { return error; } + if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } advance_root_scalar("number"); return num; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_root_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_string() noexcept { return get_string(); } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_root_raw_json_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_raw_json_string() noexcept { return get_raw_json_string(); } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_root_uint64() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64() noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer @@ -29105,10 +28513,13 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iter return NUMBER_ERROR; } auto result = numberparsing::parse_unsigned(tmpbuf); - if(result.error() == SUCCESS) { advance_root_scalar("uint64"); } + if(result.error() == SUCCESS) { + if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("uint64"); + } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_root_uint64_in_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string() noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer @@ -29117,10 +28528,13 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iter return NUMBER_ERROR; } auto result = numberparsing::parse_unsigned_in_string(tmpbuf); - if(result.error() == SUCCESS) { advance_root_scalar("uint64"); } + if(result.error() == SUCCESS) { + if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("uint64"); + } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_root_int64() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64() noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer @@ -29130,10 +28544,13 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_itera } auto result = numberparsing::parse_integer(tmpbuf); - if(result.error() == SUCCESS) { advance_root_scalar("int64"); } + if(result.error() == SUCCESS) { + if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("int64"); + } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_root_int64_in_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string() noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer @@ -29143,10 +28560,13 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_itera } auto result = numberparsing::parse_integer_in_string(tmpbuf); - if(result.error() == SUCCESS) { advance_root_scalar("int64"); } + if(result.error() == SUCCESS) { + if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("int64"); + } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_root_double() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double() noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, @@ -29158,11 +28578,14 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterat return NUMBER_ERROR; } auto result = numberparsing::parse_double(tmpbuf); - if(result.error() == SUCCESS) { advance_root_scalar("double"); } + if(result.error() == SUCCESS) { + if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("double"); + } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_root_double_in_string() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string() noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, @@ -29174,19 +28597,27 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterat return NUMBER_ERROR; } auto result = numberparsing::parse_double_in_string(tmpbuf); - if(result.error() == SUCCESS) { advance_root_scalar("double"); } + if(result.error() == SUCCESS) { + if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("double"); + } return result; } -simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::get_root_bool() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool() noexcept { auto max_len = peek_start_length(); auto json = peek_root_scalar("bool"); uint8_t tmpbuf[5+1]; if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { return incorrect_type_error("Not a boolean"); } auto result = parse_bool(tmpbuf); - if(result.error() == SUCCESS) { advance_root_scalar("bool"); } + if(result.error() == SUCCESS) { + if (!_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("bool"); + } return result; } -simdjson_really_inline bool value_iterator::is_root_null() noexcept { +simdjson_inline bool value_iterator::is_root_null() noexcept { + // If there is trailing content, then the document is not null. + if (!_json_iter->is_single_token()) { return false; } auto max_len = peek_start_length(); auto json = peek_root_scalar("null"); bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && @@ -29195,14 +28626,14 @@ simdjson_really_inline bool value_iterator::is_root_null() noexcept { return result; } -simdjson_warn_unused simdjson_really_inline error_code value_iterator::skip_child() noexcept { +simdjson_warn_unused simdjson_inline error_code value_iterator::skip_child() noexcept { SIMDJSON_ASSUME( _json_iter->token._position > _start_position ); SIMDJSON_ASSUME( _json_iter->_depth >= _depth ); return _json_iter->skip_child(depth()); } -simdjson_really_inline value_iterator value_iterator::child() const noexcept { +simdjson_inline value_iterator value_iterator::child() const noexcept { assert_at_child(); return { _json_iter, depth()+1, _json_iter->token.position() }; } @@ -29212,52 +28643,52 @@ simdjson_really_inline value_iterator value_iterator::child() const noexcept { // marked non-inline. SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING -simdjson_really_inline bool value_iterator::is_open() const noexcept { +simdjson_inline bool value_iterator::is_open() const noexcept { return _json_iter->depth() >= depth(); } SIMDJSON_POP_DISABLE_WARNINGS -simdjson_really_inline bool value_iterator::at_end() const noexcept { +simdjson_inline bool value_iterator::at_end() const noexcept { return _json_iter->at_end(); } -simdjson_really_inline bool value_iterator::at_start() const noexcept { +simdjson_inline bool value_iterator::at_start() const noexcept { return _json_iter->token.position() == start_position(); } -simdjson_really_inline bool value_iterator::at_first_field() const noexcept { +simdjson_inline bool value_iterator::at_first_field() const noexcept { SIMDJSON_ASSUME( _json_iter->token._position > _start_position ); return _json_iter->token.position() == start_position() + 1; } -simdjson_really_inline void value_iterator::abandon() noexcept { +simdjson_inline void value_iterator::abandon() noexcept { _json_iter->abandon(); } -simdjson_warn_unused simdjson_really_inline depth_t value_iterator::depth() const noexcept { +simdjson_warn_unused simdjson_inline depth_t value_iterator::depth() const noexcept { return _depth; } -simdjson_warn_unused simdjson_really_inline error_code value_iterator::error() const noexcept { +simdjson_warn_unused simdjson_inline error_code value_iterator::error() const noexcept { return _json_iter->error; } -simdjson_warn_unused simdjson_really_inline uint8_t *&value_iterator::string_buf_loc() noexcept { +simdjson_warn_unused simdjson_inline uint8_t *&value_iterator::string_buf_loc() noexcept { return _json_iter->string_buf_loc(); } -simdjson_warn_unused simdjson_really_inline const json_iterator &value_iterator::json_iter() const noexcept { +simdjson_warn_unused simdjson_inline const json_iterator &value_iterator::json_iter() const noexcept { return *_json_iter; } -simdjson_warn_unused simdjson_really_inline json_iterator &value_iterator::json_iter() noexcept { +simdjson_warn_unused simdjson_inline json_iterator &value_iterator::json_iter() noexcept { return *_json_iter; } -simdjson_really_inline const uint8_t *value_iterator::peek_start() const noexcept { +simdjson_inline const uint8_t *value_iterator::peek_start() const noexcept { return _json_iter->peek(start_position()); } -simdjson_really_inline uint32_t value_iterator::peek_start_length() const noexcept { +simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept { return _json_iter->peek_length(start_position()); } -simdjson_really_inline const uint8_t *value_iterator::peek_scalar(const char *type) noexcept { +simdjson_inline const uint8_t *value_iterator::peek_scalar(const char *type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); // If we're not at the position anymore, we don't want to advance the cursor. if (!is_at_start()) { return peek_start(); } @@ -29267,7 +28698,7 @@ simdjson_really_inline const uint8_t *value_iterator::peek_scalar(const char *ty return _json_iter->peek(); } -simdjson_really_inline void value_iterator::advance_scalar(const char *type) noexcept { +simdjson_inline void value_iterator::advance_scalar(const char *type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); // If we're not at the position anymore, we don't want to advance the cursor. if (!is_at_start()) { return; } @@ -29278,12 +28709,12 @@ simdjson_really_inline void value_iterator::advance_scalar(const char *type) noe _json_iter->ascend_to(depth()-1); } -simdjson_really_inline error_code value_iterator::start_container(uint8_t start_char, const char *incorrect_type_message, const char *type) noexcept { +simdjson_inline error_code value_iterator::start_container(uint8_t start_char, const char *incorrect_type_message, const char *type) noexcept { logger::log_start_value(*_json_iter, start_position(), depth(), type); // If we're not at the position anymore, we don't want to advance the cursor. const uint8_t *json; if (!is_at_start()) { -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS if (!is_at_iterator_start()) { return OUT_OF_ORDER_ITERATION; } #endif json = peek_start(); @@ -29305,14 +28736,14 @@ simdjson_really_inline error_code value_iterator::start_container(uint8_t start_ } -simdjson_really_inline const uint8_t *value_iterator::peek_root_scalar(const char *type) noexcept { +simdjson_inline const uint8_t *value_iterator::peek_root_scalar(const char *type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); if (!is_at_start()) { return peek_start(); } assert_at_root(); return _json_iter->peek(); } -simdjson_really_inline const uint8_t *value_iterator::peek_non_root_scalar(const char *type) noexcept { +simdjson_inline const uint8_t *value_iterator::peek_non_root_scalar(const char *type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); if (!is_at_start()) { return peek_start(); } @@ -29320,7 +28751,7 @@ simdjson_really_inline const uint8_t *value_iterator::peek_non_root_scalar(const return _json_iter->peek(); } -simdjson_really_inline void value_iterator::advance_root_scalar(const char *type) noexcept { +simdjson_inline void value_iterator::advance_root_scalar(const char *type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); if (!is_at_start()) { return; } @@ -29328,7 +28759,7 @@ simdjson_really_inline void value_iterator::advance_root_scalar(const char *type _json_iter->return_current_and_advance(); _json_iter->ascend_to(depth()-1); } -simdjson_really_inline void value_iterator::advance_non_root_scalar(const char *type) noexcept { +simdjson_inline void value_iterator::advance_non_root_scalar(const char *type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); if (!is_at_start()) { return; } @@ -29337,23 +28768,23 @@ simdjson_really_inline void value_iterator::advance_non_root_scalar(const char * _json_iter->ascend_to(depth()-1); } -simdjson_really_inline error_code value_iterator::incorrect_type_error(const char *message) const noexcept { +simdjson_inline error_code value_iterator::incorrect_type_error(const char *message) const noexcept { logger::log_error(*_json_iter, start_position(), depth(), message); return INCORRECT_TYPE; } -simdjson_really_inline bool value_iterator::is_at_start() const noexcept { +simdjson_inline bool value_iterator::is_at_start() const noexcept { return position() == start_position(); } -simdjson_really_inline bool value_iterator::is_at_key() const noexcept { +simdjson_inline bool value_iterator::is_at_key() const noexcept { // Keys are at the same depth as the object. // Note here that we could be safer and check that we are within an object, // but we do not. return _depth == _json_iter->_depth && *_json_iter->peek() == '"'; } -simdjson_really_inline bool value_iterator::is_at_iterator_start() const noexcept { +simdjson_inline bool value_iterator::is_at_iterator_start() const noexcept { // We can legitimately be either at the first value ([1]), or after the array if it's empty ([]). auto delta = position() - start_position(); return delta == 1 || delta == 2; @@ -29377,22 +28808,22 @@ inline void value_iterator::assert_at_next() const noexcept { SIMDJSON_ASSUME( _depth > 0 ); } -simdjson_really_inline void value_iterator::move_at_start() noexcept { +simdjson_inline void value_iterator::move_at_start() noexcept { _json_iter->_depth = _depth; _json_iter->token.set_position(_start_position); } -simdjson_really_inline void value_iterator::move_at_container_start() noexcept { +simdjson_inline void value_iterator::move_at_container_start() noexcept { _json_iter->_depth = _depth; _json_iter->token.set_position(_start_position + 1); } -simdjson_really_inline simdjson_result value_iterator::reset_array() noexcept { +simdjson_inline simdjson_result value_iterator::reset_array() noexcept { move_at_container_start(); return started_array(); } -simdjson_really_inline simdjson_result value_iterator::reset_object() noexcept { +simdjson_inline simdjson_result value_iterator::reset_object() noexcept { move_at_container_start(); return started_object(); } @@ -29417,11 +28848,11 @@ inline void value_iterator::assert_is_valid() const noexcept { SIMDJSON_ASSUME( _json_iter != nullptr ); } -simdjson_really_inline bool value_iterator::is_valid() const noexcept { +simdjson_inline bool value_iterator::is_valid() const noexcept { return _json_iter != nullptr; } -simdjson_really_inline simdjson_result value_iterator::type() const noexcept { +simdjson_inline simdjson_result value_iterator::type() const noexcept { switch (*peek_start()) { case '{': return json_type::object; @@ -29442,23 +28873,23 @@ simdjson_really_inline simdjson_result value_iterator::type() const n } } -simdjson_really_inline token_position value_iterator::start_position() const noexcept { +simdjson_inline token_position value_iterator::start_position() const noexcept { return _start_position; } -simdjson_really_inline token_position value_iterator::position() const noexcept { +simdjson_inline token_position value_iterator::position() const noexcept { return _json_iter->position(); } -simdjson_really_inline token_position value_iterator::end_position() const noexcept { +simdjson_inline token_position value_iterator::end_position() const noexcept { return _json_iter->end_position(); } -simdjson_really_inline token_position value_iterator::last_position() const noexcept { +simdjson_inline token_position value_iterator::last_position() const noexcept { return _json_iter->last_position(); } -simdjson_really_inline error_code value_iterator::report_error(error_code error, const char *message) noexcept { +simdjson_inline error_code value_iterator::report_error(error_code error, const char *message) noexcept { return _json_iter->report_error(error, message); } @@ -29468,9 +28899,9 @@ simdjson_really_inline error_code value_iterator::report_error(error_code error, namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value_iterator &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value_iterator &&value) noexcept : implementation_simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : implementation_simdjson_result_base(error) {} } // namespace simdjson @@ -29480,21 +28911,21 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline array_iterator::array_iterator(const value_iterator &_iter) noexcept +simdjson_inline array_iterator::array_iterator(const value_iterator &_iter) noexcept : iter{_iter} {} -simdjson_really_inline simdjson_result array_iterator::operator*() noexcept { +simdjson_inline simdjson_result array_iterator::operator*() noexcept { if (iter.error()) { iter.abandon(); return iter.error(); } return value(iter.child()); } -simdjson_really_inline bool array_iterator::operator==(const array_iterator &other) const noexcept { +simdjson_inline bool array_iterator::operator==(const array_iterator &other) const noexcept { return !(*this != other); } -simdjson_really_inline bool array_iterator::operator!=(const array_iterator &) const noexcept { +simdjson_inline bool array_iterator::operator!=(const array_iterator &) const noexcept { return iter.is_open(); } -simdjson_really_inline array_iterator &array_iterator::operator++() noexcept { +simdjson_inline array_iterator &array_iterator::operator++() noexcept { error_code error; // PERF NOTE this is a safety rail ... users should exit loops as soon as they receive an error, so we'll never get here. // However, it does not seem to make a perf difference, so we add it out of an abundance of caution. @@ -29510,31 +28941,31 @@ simdjson_really_inline array_iterator &array_iterator::operator++() noexcept { namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array_iterator &&value ) noexcept : SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base(std::forward(value)) { first.iter.assert_is_valid(); } -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : SIMDJSON_BUILTIN_IMPLEMENTATION::implementation_simdjson_result_base({}, error) { } -simdjson_really_inline simdjson_result simdjson_result::operator*() noexcept { +simdjson_inline simdjson_result simdjson_result::operator*() noexcept { if (error()) { return error(); } return *first; } -simdjson_really_inline bool simdjson_result::operator==(const simdjson_result &other) const noexcept { +simdjson_inline bool simdjson_result::operator==(const simdjson_result &other) const noexcept { if (!first.iter.is_valid()) { return !error(); } return first == other.first; } -simdjson_really_inline bool simdjson_result::operator!=(const simdjson_result &other) const noexcept { +simdjson_inline bool simdjson_result::operator!=(const simdjson_result &other) const noexcept { if (!first.iter.is_valid()) { return error(); } return first != other.first; } -simdjson_really_inline simdjson_result &simdjson_result::operator++() noexcept { +simdjson_inline simdjson_result &simdjson_result::operator++() noexcept { // Clear the error if there is one, so we don't yield it twice if (error()) { second = SUCCESS; return *this; } ++(first); @@ -29552,11 +28983,11 @@ namespace ondemand { // object_iterator // -simdjson_really_inline object_iterator::object_iterator(const value_iterator &_iter) noexcept +simdjson_inline object_iterator::object_iterator(const value_iterator &_iter) noexcept : iter{_iter} {} -simdjson_really_inline simdjson_result object_iterator::operator*() noexcept { +simdjson_inline simdjson_result object_iterator::operator*() noexcept { error_code error = iter.error(); if (error) { iter.abandon(); return error; } auto result = field::start(iter); @@ -29565,14 +28996,16 @@ simdjson_really_inline simdjson_result object_iterator::operator*() noexc if (result.error()) { iter.abandon(); } return result; } -simdjson_really_inline bool object_iterator::operator==(const object_iterator &other) const noexcept { +simdjson_inline bool object_iterator::operator==(const object_iterator &other) const noexcept { return !(*this != other); } -simdjson_really_inline bool object_iterator::operator!=(const object_iterator &) const noexcept { +simdjson_inline bool object_iterator::operator!=(const object_iterator &) const noexcept { return iter.is_open(); } -simdjson_really_inline object_iterator &object_iterator::operator++() noexcept { +SIMDJSON_PUSH_DISABLE_WARNINGS +SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING +simdjson_inline object_iterator &object_iterator::operator++() noexcept { // TODO this is a safety rail ... users should exit loops as soon as they receive an error. // Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free. if (!iter.is_open()) { return *this; } // Iterator will be released if there is an error @@ -29584,6 +29017,7 @@ simdjson_really_inline object_iterator &object_iterator::operator++() noexcept { if ((error = iter.has_next_field().get(has_value) )) { return *this; }; return *this; } +SIMDJSON_POP_DISABLE_WARNINGS // // ### Live States @@ -29632,34 +29066,34 @@ simdjson_really_inline object_iterator &object_iterator::operator++() noexcept { namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object_iterator &&value ) noexcept : implementation_simdjson_result_base(std::forward(value)) { first.iter.assert_is_valid(); } -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : implementation_simdjson_result_base({}, error) { } -simdjson_really_inline simdjson_result simdjson_result::operator*() noexcept { +simdjson_inline simdjson_result simdjson_result::operator*() noexcept { if (error()) { return error(); } return *first; } // If we're iterating and there is an error, return the error once. -simdjson_really_inline bool simdjson_result::operator==(const simdjson_result &other) const noexcept { +simdjson_inline bool simdjson_result::operator==(const simdjson_result &other) const noexcept { if (!first.iter.is_valid()) { return !error(); } return first == other.first; } // If we're iterating and there is an error, return the error once. -simdjson_really_inline bool simdjson_result::operator!=(const simdjson_result &other) const noexcept { +simdjson_inline bool simdjson_result::operator!=(const simdjson_result &other) const noexcept { if (!first.iter.is_valid()) { return error(); } return first != other.first; } // Checks for ']' and ',' -simdjson_really_inline simdjson_result &simdjson_result::operator++() noexcept { +simdjson_inline simdjson_result &simdjson_result::operator++() noexcept { // Clear the error if there is one, so we don't yield it twice if (error()) { second = SUCCESS; return *this; } ++first; @@ -29711,45 +29145,45 @@ namespace ondemand { // error == SUCCESS. // -simdjson_really_inline array::array(const value_iterator &_iter) noexcept +simdjson_inline array::array(const value_iterator &_iter) noexcept : iter{_iter} { } -simdjson_really_inline simdjson_result array::start(value_iterator &iter) noexcept { +simdjson_inline simdjson_result array::start(value_iterator &iter) noexcept { // We don't need to know if the array is empty to start iteration, but we do want to know if there // is an error--thus `simdjson_unused`. simdjson_unused bool has_value; SIMDJSON_TRY( iter.start_array().get(has_value) ); return array(iter); } -simdjson_really_inline simdjson_result array::start_root(value_iterator &iter) noexcept { +simdjson_inline simdjson_result array::start_root(value_iterator &iter) noexcept { simdjson_unused bool has_value; SIMDJSON_TRY( iter.start_root_array().get(has_value) ); return array(iter); } -simdjson_really_inline simdjson_result array::started(value_iterator &iter) noexcept { +simdjson_inline simdjson_result array::started(value_iterator &iter) noexcept { bool has_value; SIMDJSON_TRY(iter.started_array().get(has_value)); return array(iter); } -simdjson_really_inline simdjson_result array::begin() noexcept { -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +simdjson_inline simdjson_result array::begin() noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS if (!iter.is_at_iterator_start()) { return OUT_OF_ORDER_ITERATION; } #endif return array_iterator(iter); } -simdjson_really_inline simdjson_result array::end() noexcept { +simdjson_inline simdjson_result array::end() noexcept { return array_iterator(iter); } -simdjson_really_inline error_code array::consume() noexcept { +simdjson_inline error_code array::consume() noexcept { auto error = iter.json_iter().skip_child(iter.depth()-1); if(error) { iter.abandon(); } return error; } -simdjson_really_inline simdjson_result array::raw_json() noexcept { +simdjson_inline simdjson_result array::raw_json() noexcept { const uint8_t * starting_point{iter.peek_start()}; auto error = consume(); if(error) { return error; } @@ -29760,8 +29194,9 @@ simdjson_really_inline simdjson_result array::raw_json() noexc return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } +SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING -simdjson_really_inline simdjson_result array::count_elements() & noexcept { +simdjson_inline simdjson_result array::count_elements() & noexcept { size_t count{0}; // Important: we do not consume any of the values. for(simdjson_unused auto v : *this) { count++; } @@ -29772,8 +29207,9 @@ simdjson_really_inline simdjson_result array::count_elements() & noexcep iter.reset_array(); return count; } +SIMDJSON_POP_DISABLE_WARNINGS -simdjson_really_inline simdjson_result array::is_empty() & noexcept { +simdjson_inline simdjson_result array::is_empty() & noexcept { bool is_not_empty; auto error = iter.reset_array().get(is_not_empty); if(error) { return error; } @@ -29820,7 +29256,7 @@ inline simdjson_result array::at_pointer(std::string_view json_pointer) n return child; } -simdjson_really_inline simdjson_result array::at(size_t index) noexcept { +simdjson_inline simdjson_result array::at(size_t index) noexcept { size_t i = 0; for (auto value : *this) { if (i == index) { return value; } @@ -29835,7 +29271,7 @@ simdjson_really_inline simdjson_result array::at(size_t index) noexcept { namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array &&value ) noexcept : implementation_simdjson_result_base( @@ -29843,34 +29279,34 @@ simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( error_code error ) noexcept : implementation_simdjson_result_base(error) { } -simdjson_really_inline simdjson_result simdjson_result::begin() noexcept { +simdjson_inline simdjson_result simdjson_result::begin() noexcept { if (error()) { return error(); } return first.begin(); } -simdjson_really_inline simdjson_result simdjson_result::end() noexcept { +simdjson_inline simdjson_result simdjson_result::end() noexcept { if (error()) { return error(); } return first.end(); } -simdjson_really_inline simdjson_result simdjson_result::count_elements() & noexcept { +simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { if (error()) { return error(); } return first.count_elements(); } -simdjson_really_inline simdjson_result simdjson_result::is_empty() & noexcept { +simdjson_inline simdjson_result simdjson_result::is_empty() & noexcept { if (error()) { return error(); } return first.is_empty(); } -simdjson_really_inline simdjson_result simdjson_result::at(size_t index) noexcept { +simdjson_inline simdjson_result simdjson_result::at(size_t index) noexcept { if (error()) { return error(); } return first.at(index); } -simdjson_really_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { +simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { if (error()) { return error(); } return first.at_pointer(json_pointer); } @@ -29881,13 +29317,13 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline document::document(ondemand::json_iterator &&_iter) noexcept +simdjson_inline document::document(ondemand::json_iterator &&_iter) noexcept : iter{std::forward(_iter)} { logger::log_start_value(iter, "document"); } -simdjson_really_inline document document::start(json_iterator &&iter) noexcept { +simdjson_inline document document::start(json_iterator &&iter) noexcept { return document(std::forward(iter)); } @@ -29910,20 +29346,20 @@ inline int32_t document::current_depth() const noexcept { inline bool document::is_alive() noexcept { return iter.is_alive(); } -simdjson_really_inline value_iterator document::resume_value_iterator() noexcept { +simdjson_inline value_iterator document::resume_value_iterator() noexcept { return value_iterator(&iter, 1, iter.root_position()); } -simdjson_really_inline value_iterator document::get_root_value_iterator() noexcept { +simdjson_inline value_iterator document::get_root_value_iterator() noexcept { return resume_value_iterator(); } -simdjson_really_inline simdjson_result document::start_or_resume_object() noexcept { +simdjson_inline simdjson_result document::start_or_resume_object() noexcept { if (iter.at_root()) { return get_object(); } else { return object::resume(resume_value_iterator()); } } -simdjson_really_inline simdjson_result document::get_value() noexcept { +simdjson_inline simdjson_result document::get_value() noexcept { // Make sure we start any arrays or objects before returning, so that start_root_() // gets called. iter.assert_at_document_depth(); @@ -29938,139 +29374,133 @@ simdjson_really_inline simdjson_result document::get_value() noexcept { // return value(get_root_value_iterator()); } } -simdjson_really_inline simdjson_result document::get_array() & noexcept { +simdjson_inline simdjson_result document::get_array() & noexcept { auto value = get_root_value_iterator(); return array::start_root(value); } -simdjson_really_inline simdjson_result document::get_object() & noexcept { +simdjson_inline simdjson_result document::get_object() & noexcept { auto value = get_root_value_iterator(); return object::start_root(value); } -simdjson_really_inline simdjson_result document::get_uint64() noexcept { +simdjson_inline simdjson_result document::get_uint64() noexcept { return get_root_value_iterator().get_root_uint64(); } -simdjson_really_inline simdjson_result document::get_uint64_in_string() noexcept { +simdjson_inline simdjson_result document::get_uint64_in_string() noexcept { return get_root_value_iterator().get_root_uint64_in_string(); } -simdjson_really_inline simdjson_result document::get_int64() noexcept { +simdjson_inline simdjson_result document::get_int64() noexcept { return get_root_value_iterator().get_root_int64(); } -simdjson_really_inline simdjson_result document::get_int64_in_string() noexcept { +simdjson_inline simdjson_result document::get_int64_in_string() noexcept { return get_root_value_iterator().get_root_int64_in_string(); } -simdjson_really_inline simdjson_result document::get_double() noexcept { +simdjson_inline simdjson_result document::get_double() noexcept { return get_root_value_iterator().get_root_double(); } -simdjson_really_inline simdjson_result document::get_double_in_string() noexcept { +simdjson_inline simdjson_result document::get_double_in_string() noexcept { return get_root_value_iterator().get_root_double_in_string(); } -simdjson_really_inline simdjson_result document::get_string() noexcept { +simdjson_inline simdjson_result document::get_string() noexcept { return get_root_value_iterator().get_root_string(); } -simdjson_really_inline simdjson_result document::get_raw_json_string() noexcept { +simdjson_inline simdjson_result document::get_raw_json_string() noexcept { return get_root_value_iterator().get_root_raw_json_string(); } -simdjson_really_inline simdjson_result document::get_bool() noexcept { +simdjson_inline simdjson_result document::get_bool() noexcept { return get_root_value_iterator().get_root_bool(); } -simdjson_really_inline bool document::is_null() noexcept { +simdjson_inline simdjson_result document::is_null() noexcept { return get_root_value_iterator().is_root_null(); } -template<> simdjson_really_inline simdjson_result document::get() & noexcept { return get_array(); } -template<> simdjson_really_inline simdjson_result document::get() & noexcept { return get_object(); } -template<> simdjson_really_inline simdjson_result document::get() & noexcept { return get_raw_json_string(); } -template<> simdjson_really_inline simdjson_result document::get() & noexcept { return get_string(); } -template<> simdjson_really_inline simdjson_result document::get() & noexcept { return get_double(); } -template<> simdjson_really_inline simdjson_result document::get() & noexcept { return get_uint64(); } -template<> simdjson_really_inline simdjson_result document::get() & noexcept { return get_int64(); } -template<> simdjson_really_inline simdjson_result document::get() & noexcept { return get_bool(); } -template<> simdjson_really_inline simdjson_result document::get() & noexcept { return get_value(); } - -template<> simdjson_really_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } -template<> simdjson_really_inline simdjson_result document::get() && noexcept { return get_string(); } -template<> simdjson_really_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } -template<> simdjson_really_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } -template<> simdjson_really_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } -template<> simdjson_really_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } -template<> simdjson_really_inline simdjson_result document::get() && noexcept { return get_value(); } - -template simdjson_really_inline error_code document::get(T &out) & noexcept { +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_array(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_object(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_raw_json_string(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_string(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_double(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_uint64(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_int64(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_bool(); } +template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } + +template<> simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } +template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(); } +template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } +template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } +template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } +template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } +template<> simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } + +template simdjson_inline error_code document::get(T &out) & noexcept { return get().get(out); } -template simdjson_really_inline error_code document::get(T &out) && noexcept { +template simdjson_inline error_code document::get(T &out) && noexcept { return std::forward(*this).get().get(out); } #if SIMDJSON_EXCEPTIONS -simdjson_really_inline document::operator array() & noexcept(false) { return get_array(); } -simdjson_really_inline document::operator object() & noexcept(false) { return get_object(); } -simdjson_really_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } -simdjson_really_inline document::operator int64_t() noexcept(false) { return get_int64(); } -simdjson_really_inline document::operator double() noexcept(false) { return get_double(); } -simdjson_really_inline document::operator std::string_view() noexcept(false) { return get_string(); } -simdjson_really_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } -simdjson_really_inline document::operator bool() noexcept(false) { return get_bool(); } -simdjson_really_inline document::operator value() noexcept(false) { return get_value(); } +simdjson_inline document::operator array() & noexcept(false) { return get_array(); } +simdjson_inline document::operator object() & noexcept(false) { return get_object(); } +simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } +simdjson_inline document::operator int64_t() noexcept(false) { return get_int64(); } +simdjson_inline document::operator double() noexcept(false) { return get_double(); } +simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(); } +simdjson_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } +simdjson_inline document::operator bool() noexcept(false) { return get_bool(); } +simdjson_inline document::operator value() noexcept(false) { return get_value(); } #endif -simdjson_really_inline simdjson_result document::count_elements() & noexcept { +simdjson_inline simdjson_result document::count_elements() & noexcept { auto a = get_array(); simdjson_result answer = a.count_elements(); /* If there was an array, we are now left pointing at its first element. */ - if(answer.error() == SUCCESS) { - iter._depth = 1 ; /* undoing the increment so we go back at the doc depth.*/ - iter.assert_at_document_depth(); - } + if(answer.error() == SUCCESS) { rewind(); } return answer; } -simdjson_really_inline simdjson_result document::count_fields() & noexcept { +simdjson_inline simdjson_result document::count_fields() & noexcept { auto a = get_object(); simdjson_result answer = a.count_fields(); - /* If there was an array, we are now left pointing at its first element. */ - if(answer.error() == SUCCESS) { - iter._depth = 1 ; /* undoing the increment so we go back at the doc depth.*/ - iter.assert_at_document_depth(); - } + /* If there was an object, we are now left pointing at its first element. */ + if(answer.error() == SUCCESS) { rewind(); } return answer; } -simdjson_really_inline simdjson_result document::at(size_t index) & noexcept { +simdjson_inline simdjson_result document::at(size_t index) & noexcept { auto a = get_array(); return a.at(index); } -simdjson_really_inline simdjson_result document::begin() & noexcept { +simdjson_inline simdjson_result document::begin() & noexcept { return get_array().begin(); } -simdjson_really_inline simdjson_result document::end() & noexcept { +simdjson_inline simdjson_result document::end() & noexcept { return {}; } -simdjson_really_inline simdjson_result document::find_field(std::string_view key) & noexcept { +simdjson_inline simdjson_result document::find_field(std::string_view key) & noexcept { return start_or_resume_object().find_field(key); } -simdjson_really_inline simdjson_result document::find_field(const char *key) & noexcept { +simdjson_inline simdjson_result document::find_field(const char *key) & noexcept { return start_or_resume_object().find_field(key); } -simdjson_really_inline simdjson_result document::find_field_unordered(std::string_view key) & noexcept { +simdjson_inline simdjson_result document::find_field_unordered(std::string_view key) & noexcept { return start_or_resume_object().find_field_unordered(key); } -simdjson_really_inline simdjson_result document::find_field_unordered(const char *key) & noexcept { +simdjson_inline simdjson_result document::find_field_unordered(const char *key) & noexcept { return start_or_resume_object().find_field_unordered(key); } -simdjson_really_inline simdjson_result document::operator[](std::string_view key) & noexcept { +simdjson_inline simdjson_result document::operator[](std::string_view key) & noexcept { return start_or_resume_object()[key]; } -simdjson_really_inline simdjson_result document::operator[](const char *key) & noexcept { +simdjson_inline simdjson_result document::operator[](const char *key) & noexcept { return start_or_resume_object()[key]; } -simdjson_really_inline error_code document::consume() noexcept { +simdjson_inline error_code document::consume() noexcept { auto error = iter.skip_child(0); if(error) { iter.abandon(); } return error; } -simdjson_really_inline simdjson_result document::raw_json() noexcept { +simdjson_inline simdjson_result document::raw_json() noexcept { auto _iter = get_root_value_iterator(); const uint8_t * starting_point{_iter.peek_start()}; auto error = consume(); @@ -30082,40 +29512,40 @@ simdjson_really_inline simdjson_result document::raw_json() no return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } -simdjson_really_inline simdjson_result document::type() noexcept { +simdjson_inline simdjson_result document::type() noexcept { return get_root_value_iterator().type(); } -simdjson_really_inline simdjson_result document::is_scalar() noexcept { +simdjson_inline simdjson_result document::is_scalar() noexcept { json_type this_type; auto error = type().get(this_type); if(error) { return error; } return ! ((this_type == json_type::array) || (this_type == json_type::object)); } -simdjson_really_inline bool document::is_negative() noexcept { +simdjson_inline bool document::is_negative() noexcept { return get_root_value_iterator().is_root_negative(); } -simdjson_really_inline simdjson_result document::is_integer() noexcept { +simdjson_inline simdjson_result document::is_integer() noexcept { return get_root_value_iterator().is_root_integer(); } -simdjson_really_inline simdjson_result document::get_number_type() noexcept { +simdjson_inline simdjson_result document::get_number_type() noexcept { return get_root_value_iterator().get_root_number_type(); } -simdjson_really_inline simdjson_result document::get_number() noexcept { +simdjson_inline simdjson_result document::get_number() noexcept { return get_root_value_iterator().get_root_number(); } -simdjson_really_inline simdjson_result document::raw_json_token() noexcept { +simdjson_inline simdjson_result document::raw_json_token() noexcept { auto _iter = get_root_value_iterator(); return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_start_length()); } -simdjson_really_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { +simdjson_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { rewind(); // Rewind the document each time at_pointer is called if (json_pointer.empty()) { return this->get_value(); @@ -30139,7 +29569,7 @@ simdjson_really_inline simdjson_result document::at_pointer(std::string_v namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document &&value ) noexcept : implementation_simdjson_result_base( @@ -30147,7 +29577,7 @@ simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( error_code error ) noexcept : implementation_simdjson_result_base( @@ -30155,216 +29585,216 @@ simdjson_really_inline simdjson_result simdjson_result::count_elements() & noexcept { +simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { if (error()) { return error(); } return first.count_elements(); } -simdjson_really_inline simdjson_result simdjson_result::count_fields() & noexcept { +simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { if (error()) { return error(); } return first.count_fields(); } -simdjson_really_inline simdjson_result simdjson_result::at(size_t index) & noexcept { +simdjson_inline simdjson_result simdjson_result::at(size_t index) & noexcept { if (error()) { return error(); } return first.at(index); } -simdjson_really_inline error_code simdjson_result::rewind() noexcept { +simdjson_inline error_code simdjson_result::rewind() noexcept { if (error()) { return error(); } first.rewind(); return SUCCESS; } -simdjson_really_inline simdjson_result simdjson_result::begin() & noexcept { +simdjson_inline simdjson_result simdjson_result::begin() & noexcept { if (error()) { return error(); } return first.begin(); } -simdjson_really_inline simdjson_result simdjson_result::end() & noexcept { +simdjson_inline simdjson_result simdjson_result::end() & noexcept { return {}; } -simdjson_really_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { if (error()) { return error(); } return first.find_field_unordered(key); } -simdjson_really_inline simdjson_result simdjson_result::find_field_unordered(const char *key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field_unordered(const char *key) & noexcept { if (error()) { return error(); } return first.find_field_unordered(key); } -simdjson_really_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { if (error()) { return error(); } return first[key]; } -simdjson_really_inline simdjson_result simdjson_result::operator[](const char *key) & noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](const char *key) & noexcept { if (error()) { return error(); } return first[key]; } -simdjson_really_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { if (error()) { return error(); } return first.find_field(key); } -simdjson_really_inline simdjson_result simdjson_result::find_field(const char *key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field(const char *key) & noexcept { if (error()) { return error(); } return first.find_field(key); } -simdjson_really_inline simdjson_result simdjson_result::get_array() & noexcept { +simdjson_inline simdjson_result simdjson_result::get_array() & noexcept { if (error()) { return error(); } return first.get_array(); } -simdjson_really_inline simdjson_result simdjson_result::get_object() & noexcept { +simdjson_inline simdjson_result simdjson_result::get_object() & noexcept { if (error()) { return error(); } return first.get_object(); } -simdjson_really_inline simdjson_result simdjson_result::get_uint64() noexcept { +simdjson_inline simdjson_result simdjson_result::get_uint64() noexcept { if (error()) { return error(); } return first.get_uint64(); } -simdjson_really_inline simdjson_result simdjson_result::get_int64() noexcept { +simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { if (error()) { return error(); } return first.get_int64(); } -simdjson_really_inline simdjson_result simdjson_result::get_double() noexcept { +simdjson_inline simdjson_result simdjson_result::get_double() noexcept { if (error()) { return error(); } return first.get_double(); } -simdjson_really_inline simdjson_result simdjson_result::get_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_string() noexcept { if (error()) { return error(); } return first.get_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { if (error()) { return error(); } return first.get_raw_json_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_bool() noexcept { +simdjson_inline simdjson_result simdjson_result::get_bool() noexcept { if (error()) { return error(); } return first.get_bool(); } -simdjson_really_inline simdjson_result simdjson_result::get_value() noexcept { +simdjson_inline simdjson_result simdjson_result::get_value() noexcept { if (error()) { return error(); } return first.get_value(); } -simdjson_really_inline bool simdjson_result::is_null() noexcept { +simdjson_inline simdjson_result simdjson_result::is_null() noexcept { if (error()) { return error(); } return first.is_null(); } template -simdjson_really_inline simdjson_result simdjson_result::get() & noexcept { +simdjson_inline simdjson_result simdjson_result::get() & noexcept { if (error()) { return error(); } return first.get(); } template -simdjson_really_inline simdjson_result simdjson_result::get() && noexcept { +simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first).get(); } template -simdjson_really_inline error_code simdjson_result::get(T &out) & noexcept { +simdjson_inline error_code simdjson_result::get(T &out) & noexcept { if (error()) { return error(); } return first.get(out); } template -simdjson_really_inline error_code simdjson_result::get(T &out) && noexcept { +simdjson_inline error_code simdjson_result::get(T &out) && noexcept { if (error()) { return error(); } return std::forward(first).get(out); } -template<> simdjson_really_inline simdjson_result simdjson_result::get() & noexcept = delete; -template<> simdjson_really_inline simdjson_result simdjson_result::get() && noexcept { +template<> simdjson_inline simdjson_result simdjson_result::get() & noexcept = delete; +template<> simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first); } -template<> simdjson_really_inline error_code simdjson_result::get(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document &out) & noexcept = delete; -template<> simdjson_really_inline error_code simdjson_result::get(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document &out) && noexcept { +template<> simdjson_inline error_code simdjson_result::get(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document &out) & noexcept = delete; +template<> simdjson_inline error_code simdjson_result::get(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document &out) && noexcept { if (error()) { return error(); } out = std::forward(first); return SUCCESS; } -simdjson_really_inline simdjson_result simdjson_result::type() noexcept { +simdjson_inline simdjson_result simdjson_result::type() noexcept { if (error()) { return error(); } return first.type(); } -simdjson_really_inline simdjson_result simdjson_result::is_scalar() noexcept { +simdjson_inline simdjson_result simdjson_result::is_scalar() noexcept { if (error()) { return error(); } return first.is_scalar(); } -simdjson_really_inline bool simdjson_result::is_negative() noexcept { +simdjson_inline bool simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); } -simdjson_really_inline simdjson_result simdjson_result::is_integer() noexcept { +simdjson_inline simdjson_result simdjson_result::is_integer() noexcept { if (error()) { return error(); } return first.is_integer(); } -simdjson_really_inline simdjson_result simdjson_result::get_number_type() noexcept { +simdjson_inline simdjson_result simdjson_result::get_number_type() noexcept { if (error()) { return error(); } return first.get_number_type(); } -simdjson_really_inline simdjson_result simdjson_result::get_number() noexcept { +simdjson_inline simdjson_result simdjson_result::get_number() noexcept { if (error()) { return error(); } return first.get_number(); } #if SIMDJSON_EXCEPTIONS -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() & noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() & noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator uint64_t() noexcept(false) { +simdjson_inline simdjson_result::operator uint64_t() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator int64_t() noexcept(false) { +simdjson_inline simdjson_result::operator int64_t() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator double() noexcept(false) { +simdjson_inline simdjson_result::operator double() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator std::string_view() noexcept(false) { +simdjson_inline simdjson_result::operator std::string_view() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator bool() noexcept(false) { +simdjson_inline simdjson_result::operator bool() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value() noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } #endif -simdjson_really_inline simdjson_result simdjson_result::current_location() noexcept { +simdjson_inline simdjson_result simdjson_result::current_location() noexcept { if (error()) { return error(); } return first.current_location(); } -simdjson_really_inline int32_t simdjson_result::current_depth() const noexcept { +simdjson_inline int32_t simdjson_result::current_depth() const noexcept { if (error()) { return error(); } return first.current_depth(); } -simdjson_really_inline simdjson_result simdjson_result::raw_json_token() noexcept { +simdjson_inline simdjson_result simdjson_result::raw_json_token() noexcept { if (error()) { return error(); } return first.raw_json_token(); } -simdjson_really_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { +simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { if (error()) { return error(); } return first.at_pointer(json_pointer); } @@ -30377,54 +29807,54 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline document_reference::document_reference() noexcept : doc{nullptr} {} -simdjson_really_inline document_reference::document_reference(document &d) noexcept : doc(&d) {} -simdjson_really_inline void document_reference::rewind() noexcept { doc->rewind(); } -simdjson_really_inline simdjson_result document_reference::get_array() & noexcept { return doc->get_array(); } -simdjson_really_inline simdjson_result document_reference::get_object() & noexcept { return doc->get_object(); } -simdjson_really_inline simdjson_result document_reference::get_uint64() noexcept { return doc->get_uint64(); } -simdjson_really_inline simdjson_result document_reference::get_int64() noexcept { return doc->get_int64(); } -simdjson_really_inline simdjson_result document_reference::get_double() noexcept { return doc->get_double(); } -simdjson_really_inline simdjson_result document_reference::get_string() noexcept { return doc->get_string(); } -simdjson_really_inline simdjson_result document_reference::get_raw_json_string() noexcept { return doc->get_raw_json_string(); } -simdjson_really_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_bool(); } -simdjson_really_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } -simdjson_really_inline bool document_reference::is_null() noexcept { return doc->is_null(); } +simdjson_inline document_reference::document_reference() noexcept : doc{nullptr} {} +simdjson_inline document_reference::document_reference(document &d) noexcept : doc(&d) {} +simdjson_inline void document_reference::rewind() noexcept { doc->rewind(); } +simdjson_inline simdjson_result document_reference::get_array() & noexcept { return doc->get_array(); } +simdjson_inline simdjson_result document_reference::get_object() & noexcept { return doc->get_object(); } +simdjson_inline simdjson_result document_reference::get_uint64() noexcept { return doc->get_uint64(); } +simdjson_inline simdjson_result document_reference::get_int64() noexcept { return doc->get_int64(); } +simdjson_inline simdjson_result document_reference::get_double() noexcept { return doc->get_double(); } +simdjson_inline simdjson_result document_reference::get_string() noexcept { return doc->get_string(); } +simdjson_inline simdjson_result document_reference::get_raw_json_string() noexcept { return doc->get_raw_json_string(); } +simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_bool(); } +simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } +simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->is_null(); } #if SIMDJSON_EXCEPTIONS -simdjson_really_inline document_reference::operator array() & noexcept(false) { return array(*doc); } -simdjson_really_inline document_reference::operator object() & noexcept(false) { return object(*doc); } -simdjson_really_inline document_reference::operator uint64_t() noexcept(false) { return uint64_t(*doc); } -simdjson_really_inline document_reference::operator int64_t() noexcept(false) { return int64_t(*doc); } -simdjson_really_inline document_reference::operator double() noexcept(false) { return double(*doc); } -simdjson_really_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } -simdjson_really_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); } -simdjson_really_inline document_reference::operator bool() noexcept(false) { return bool(*doc); } -simdjson_really_inline document_reference::operator value() noexcept(false) { return value(*doc); } +simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } +simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } +simdjson_inline document_reference::operator uint64_t() noexcept(false) { return uint64_t(*doc); } +simdjson_inline document_reference::operator int64_t() noexcept(false) { return int64_t(*doc); } +simdjson_inline document_reference::operator double() noexcept(false) { return double(*doc); } +simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } +simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); } +simdjson_inline document_reference::operator bool() noexcept(false) { return bool(*doc); } +simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } #endif -simdjson_really_inline simdjson_result document_reference::count_elements() & noexcept { return doc->count_elements(); } -simdjson_really_inline simdjson_result document_reference::count_fields() & noexcept { return doc->count_fields(); } -simdjson_really_inline simdjson_result document_reference::at(size_t index) & noexcept { return doc->at(index); } -simdjson_really_inline simdjson_result document_reference::begin() & noexcept { return doc->begin(); } -simdjson_really_inline simdjson_result document_reference::end() & noexcept { return doc->end(); } -simdjson_really_inline simdjson_result document_reference::find_field(std::string_view key) & noexcept { return doc->find_field(key); } -simdjson_really_inline simdjson_result document_reference::find_field(const char *key) & noexcept { return doc->find_field(key); } -simdjson_really_inline simdjson_result document_reference::operator[](std::string_view key) & noexcept { return (*doc)[key]; } -simdjson_really_inline simdjson_result document_reference::operator[](const char *key) & noexcept { return (*doc)[key]; } -simdjson_really_inline simdjson_result document_reference::find_field_unordered(std::string_view key) & noexcept { return doc->find_field_unordered(key); } -simdjson_really_inline simdjson_result document_reference::find_field_unordered(const char *key) & noexcept { return doc->find_field_unordered(key); } -simdjson_really_inline simdjson_result document_reference::type() noexcept { return doc->type(); } -simdjson_really_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } -simdjson_really_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } -simdjson_really_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } -simdjson_really_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } -simdjson_really_inline simdjson_result document_reference::is_integer() noexcept { return doc->is_integer(); } -simdjson_really_inline simdjson_result document_reference::get_number_type() noexcept { return doc->get_number_type(); } -simdjson_really_inline simdjson_result document_reference::get_number() noexcept { return doc->get_number(); } -simdjson_really_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } -simdjson_really_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } -simdjson_really_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} -simdjson_really_inline document_reference::operator document&() const noexcept { return *doc; } +simdjson_inline simdjson_result document_reference::count_elements() & noexcept { return doc->count_elements(); } +simdjson_inline simdjson_result document_reference::count_fields() & noexcept { return doc->count_fields(); } +simdjson_inline simdjson_result document_reference::at(size_t index) & noexcept { return doc->at(index); } +simdjson_inline simdjson_result document_reference::begin() & noexcept { return doc->begin(); } +simdjson_inline simdjson_result document_reference::end() & noexcept { return doc->end(); } +simdjson_inline simdjson_result document_reference::find_field(std::string_view key) & noexcept { return doc->find_field(key); } +simdjson_inline simdjson_result document_reference::find_field(const char *key) & noexcept { return doc->find_field(key); } +simdjson_inline simdjson_result document_reference::operator[](std::string_view key) & noexcept { return (*doc)[key]; } +simdjson_inline simdjson_result document_reference::operator[](const char *key) & noexcept { return (*doc)[key]; } +simdjson_inline simdjson_result document_reference::find_field_unordered(std::string_view key) & noexcept { return doc->find_field_unordered(key); } +simdjson_inline simdjson_result document_reference::find_field_unordered(const char *key) & noexcept { return doc->find_field_unordered(key); } +simdjson_inline simdjson_result document_reference::type() noexcept { return doc->type(); } +simdjson_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } +simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } +simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } +simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } +simdjson_inline simdjson_result document_reference::is_integer() noexcept { return doc->is_integer(); } +simdjson_inline simdjson_result document_reference::get_number_type() noexcept { return doc->get_number_type(); } +simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_number(); } +simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } +simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } +simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json();} +simdjson_inline document_reference::operator document&() const noexcept { return *doc; } } // namespace ondemand } // namespace SIMDJSON_BUILTIN_IMPLEMENTATION @@ -30433,172 +29863,172 @@ simdjson_really_inline document_reference::operator document&() const noexcept { namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_reference value, error_code error) +simdjson_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_reference value, error_code error) noexcept : implementation_simdjson_result_base(std::forward(value), error) {} -simdjson_really_inline simdjson_result simdjson_result::count_elements() & noexcept { +simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { if (error()) { return error(); } return first.count_elements(); } -simdjson_really_inline simdjson_result simdjson_result::count_fields() & noexcept { +simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { if (error()) { return error(); } return first.count_fields(); } -simdjson_really_inline simdjson_result simdjson_result::at(size_t index) & noexcept { +simdjson_inline simdjson_result simdjson_result::at(size_t index) & noexcept { if (error()) { return error(); } return first.at(index); } -simdjson_really_inline error_code simdjson_result::rewind() noexcept { +simdjson_inline error_code simdjson_result::rewind() noexcept { if (error()) { return error(); } first.rewind(); return SUCCESS; } -simdjson_really_inline simdjson_result simdjson_result::begin() & noexcept { +simdjson_inline simdjson_result simdjson_result::begin() & noexcept { if (error()) { return error(); } return first.begin(); } -simdjson_really_inline simdjson_result simdjson_result::end() & noexcept { +simdjson_inline simdjson_result simdjson_result::end() & noexcept { return {}; } -simdjson_really_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { if (error()) { return error(); } return first.find_field_unordered(key); } -simdjson_really_inline simdjson_result simdjson_result::find_field_unordered(const char *key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field_unordered(const char *key) & noexcept { if (error()) { return error(); } return first.find_field_unordered(key); } -simdjson_really_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { if (error()) { return error(); } return first[key]; } -simdjson_really_inline simdjson_result simdjson_result::operator[](const char *key) & noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](const char *key) & noexcept { if (error()) { return error(); } return first[key]; } -simdjson_really_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { if (error()) { return error(); } return first.find_field(key); } -simdjson_really_inline simdjson_result simdjson_result::find_field(const char *key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field(const char *key) & noexcept { if (error()) { return error(); } return first.find_field(key); } -simdjson_really_inline simdjson_result simdjson_result::get_array() & noexcept { +simdjson_inline simdjson_result simdjson_result::get_array() & noexcept { if (error()) { return error(); } return first.get_array(); } -simdjson_really_inline simdjson_result simdjson_result::get_object() & noexcept { +simdjson_inline simdjson_result simdjson_result::get_object() & noexcept { if (error()) { return error(); } return first.get_object(); } -simdjson_really_inline simdjson_result simdjson_result::get_uint64() noexcept { +simdjson_inline simdjson_result simdjson_result::get_uint64() noexcept { if (error()) { return error(); } return first.get_uint64(); } -simdjson_really_inline simdjson_result simdjson_result::get_int64() noexcept { +simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { if (error()) { return error(); } return first.get_int64(); } -simdjson_really_inline simdjson_result simdjson_result::get_double() noexcept { +simdjson_inline simdjson_result simdjson_result::get_double() noexcept { if (error()) { return error(); } return first.get_double(); } -simdjson_really_inline simdjson_result simdjson_result::get_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_string() noexcept { if (error()) { return error(); } return first.get_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { if (error()) { return error(); } return first.get_raw_json_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_bool() noexcept { +simdjson_inline simdjson_result simdjson_result::get_bool() noexcept { if (error()) { return error(); } return first.get_bool(); } -simdjson_really_inline simdjson_result simdjson_result::get_value() noexcept { +simdjson_inline simdjson_result simdjson_result::get_value() noexcept { if (error()) { return error(); } return first.get_value(); } -simdjson_really_inline bool simdjson_result::is_null() noexcept { +simdjson_inline simdjson_result simdjson_result::is_null() noexcept { if (error()) { return error(); } return first.is_null(); } -simdjson_really_inline simdjson_result simdjson_result::type() noexcept { +simdjson_inline simdjson_result simdjson_result::type() noexcept { if (error()) { return error(); } return first.type(); } -simdjson_really_inline simdjson_result simdjson_result::is_scalar() noexcept { +simdjson_inline simdjson_result simdjson_result::is_scalar() noexcept { if (error()) { return error(); } return first.is_scalar(); } -simdjson_really_inline bool simdjson_result::is_negative() noexcept { +simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); } -simdjson_really_inline simdjson_result simdjson_result::is_integer() noexcept { +simdjson_inline simdjson_result simdjson_result::is_integer() noexcept { if (error()) { return error(); } return first.is_integer(); } -simdjson_really_inline simdjson_result simdjson_result::get_number_type() noexcept { +simdjson_inline simdjson_result simdjson_result::get_number_type() noexcept { if (error()) { return error(); } return first.get_number_type(); } -simdjson_really_inline simdjson_result simdjson_result::get_number() noexcept { +simdjson_inline simdjson_result simdjson_result::get_number() noexcept { if (error()) { return error(); } return first.get_number(); } #if SIMDJSON_EXCEPTIONS -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() & noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() & noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator uint64_t() noexcept(false) { +simdjson_inline simdjson_result::operator uint64_t() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator int64_t() noexcept(false) { +simdjson_inline simdjson_result::operator int64_t() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator double() noexcept(false) { +simdjson_inline simdjson_result::operator double() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator std::string_view() noexcept(false) { +simdjson_inline simdjson_result::operator std::string_view() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator bool() noexcept(false) { +simdjson_inline simdjson_result::operator bool() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value() noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } #endif -simdjson_really_inline simdjson_result simdjson_result::current_location() noexcept { +simdjson_inline simdjson_result simdjson_result::current_location() noexcept { if (error()) { return error(); } return first.current_location(); } -simdjson_really_inline simdjson_result simdjson_result::raw_json_token() noexcept { +simdjson_inline simdjson_result simdjson_result::raw_json_token() noexcept { if (error()) { return error(); } return first.raw_json_token(); } -simdjson_really_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { +simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { if (error()) { return error(); } return first.at_pointer(json_pointer); } @@ -30611,24 +30041,24 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline value::value(const value_iterator &_iter) noexcept +simdjson_inline value::value(const value_iterator &_iter) noexcept : iter{_iter} { } -simdjson_really_inline value value::start(const value_iterator &iter) noexcept { +simdjson_inline value value::start(const value_iterator &iter) noexcept { return iter; } -simdjson_really_inline value value::resume(const value_iterator &iter) noexcept { +simdjson_inline value value::resume(const value_iterator &iter) noexcept { return iter; } -simdjson_really_inline simdjson_result value::get_array() noexcept { +simdjson_inline simdjson_result value::get_array() noexcept { return array::start(iter); } -simdjson_really_inline simdjson_result value::get_object() noexcept { +simdjson_inline simdjson_result value::get_object() noexcept { return object::start(iter); } -simdjson_really_inline simdjson_result value::start_or_resume_object() noexcept { +simdjson_inline simdjson_result value::start_or_resume_object() noexcept { if (iter.at_start()) { return get_object(); } else { @@ -30636,85 +30066,84 @@ simdjson_really_inline simdjson_result value::start_or_resume_object() n } } -simdjson_really_inline simdjson_result value::get_raw_json_string() noexcept { +simdjson_inline simdjson_result value::get_raw_json_string() noexcept { return iter.get_raw_json_string(); } -simdjson_really_inline simdjson_result value::get_string() noexcept { +simdjson_inline simdjson_result value::get_string() noexcept { return iter.get_string(); } -simdjson_really_inline simdjson_result value::get_double() noexcept { +simdjson_inline simdjson_result value::get_double() noexcept { return iter.get_double(); } -simdjson_really_inline simdjson_result value::get_double_in_string() noexcept { +simdjson_inline simdjson_result value::get_double_in_string() noexcept { return iter.get_double_in_string(); } -simdjson_really_inline simdjson_result value::get_uint64() noexcept { +simdjson_inline simdjson_result value::get_uint64() noexcept { return iter.get_uint64(); } -simdjson_really_inline simdjson_result value::get_uint64_in_string() noexcept { +simdjson_inline simdjson_result value::get_uint64_in_string() noexcept { return iter.get_uint64_in_string(); } -simdjson_really_inline simdjson_result value::get_int64() noexcept { +simdjson_inline simdjson_result value::get_int64() noexcept { return iter.get_int64(); } -simdjson_really_inline simdjson_result value::get_int64_in_string() noexcept { +simdjson_inline simdjson_result value::get_int64_in_string() noexcept { return iter.get_int64_in_string(); } -simdjson_really_inline simdjson_result value::get_bool() noexcept { +simdjson_inline simdjson_result value::get_bool() noexcept { return iter.get_bool(); } -simdjson_really_inline bool value::is_null() noexcept { +simdjson_inline simdjson_result value::is_null() noexcept { return iter.is_null(); } - -template<> simdjson_really_inline simdjson_result value::get() noexcept { return get_array(); } -template<> simdjson_really_inline simdjson_result value::get() noexcept { return get_object(); } -template<> simdjson_really_inline simdjson_result value::get() noexcept { return get_raw_json_string(); } -template<> simdjson_really_inline simdjson_result value::get() noexcept { return get_string(); } -template<> simdjson_really_inline simdjson_result value::get() noexcept { return get_number(); } -template<> simdjson_really_inline simdjson_result value::get() noexcept { return get_double(); } -template<> simdjson_really_inline simdjson_result value::get() noexcept { return get_uint64(); } -template<> simdjson_really_inline simdjson_result value::get() noexcept { return get_int64(); } -template<> simdjson_really_inline simdjson_result value::get() noexcept { return get_bool(); } - -template simdjson_really_inline error_code value::get(T &out) noexcept { +template<> simdjson_inline simdjson_result value::get() noexcept { return get_array(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_object(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_raw_json_string(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_string(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_number(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_double(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_uint64(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_int64(); } +template<> simdjson_inline simdjson_result value::get() noexcept { return get_bool(); } + +template simdjson_inline error_code value::get(T &out) noexcept { return get().get(out); } #if SIMDJSON_EXCEPTIONS -simdjson_really_inline value::operator array() noexcept(false) { +simdjson_inline value::operator array() noexcept(false) { return get_array(); } -simdjson_really_inline value::operator object() noexcept(false) { +simdjson_inline value::operator object() noexcept(false) { return get_object(); } -simdjson_really_inline value::operator uint64_t() noexcept(false) { +simdjson_inline value::operator uint64_t() noexcept(false) { return get_uint64(); } -simdjson_really_inline value::operator int64_t() noexcept(false) { +simdjson_inline value::operator int64_t() noexcept(false) { return get_int64(); } -simdjson_really_inline value::operator double() noexcept(false) { +simdjson_inline value::operator double() noexcept(false) { return get_double(); } -simdjson_really_inline value::operator std::string_view() noexcept(false) { +simdjson_inline value::operator std::string_view() noexcept(false) { return get_string(); } -simdjson_really_inline value::operator raw_json_string() noexcept(false) { +simdjson_inline value::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } -simdjson_really_inline value::operator bool() noexcept(false) { +simdjson_inline value::operator bool() noexcept(false) { return get_bool(); } #endif -simdjson_really_inline simdjson_result value::begin() & noexcept { +simdjson_inline simdjson_result value::begin() & noexcept { return get_array().begin(); } -simdjson_really_inline simdjson_result value::end() & noexcept { +simdjson_inline simdjson_result value::end() & noexcept { return {}; } -simdjson_really_inline simdjson_result value::count_elements() & noexcept { +simdjson_inline simdjson_result value::count_elements() & noexcept { simdjson_result answer; auto a = get_array(); answer = a.count_elements(); @@ -30724,77 +30153,77 @@ simdjson_really_inline simdjson_result value::count_elements() & noexcep iter.move_at_start(); return answer; } -simdjson_really_inline simdjson_result value::count_fields() & noexcept { +simdjson_inline simdjson_result value::count_fields() & noexcept { simdjson_result answer; auto a = get_object(); answer = a.count_fields(); iter.move_at_start(); return answer; } -simdjson_really_inline simdjson_result value::at(size_t index) noexcept { +simdjson_inline simdjson_result value::at(size_t index) noexcept { auto a = get_array(); return a.at(index); } -simdjson_really_inline simdjson_result value::find_field(std::string_view key) noexcept { +simdjson_inline simdjson_result value::find_field(std::string_view key) noexcept { return start_or_resume_object().find_field(key); } -simdjson_really_inline simdjson_result value::find_field(const char *key) noexcept { +simdjson_inline simdjson_result value::find_field(const char *key) noexcept { return start_or_resume_object().find_field(key); } -simdjson_really_inline simdjson_result value::find_field_unordered(std::string_view key) noexcept { +simdjson_inline simdjson_result value::find_field_unordered(std::string_view key) noexcept { return start_or_resume_object().find_field_unordered(key); } -simdjson_really_inline simdjson_result value::find_field_unordered(const char *key) noexcept { +simdjson_inline simdjson_result value::find_field_unordered(const char *key) noexcept { return start_or_resume_object().find_field_unordered(key); } -simdjson_really_inline simdjson_result value::operator[](std::string_view key) noexcept { +simdjson_inline simdjson_result value::operator[](std::string_view key) noexcept { return start_or_resume_object()[key]; } -simdjson_really_inline simdjson_result value::operator[](const char *key) noexcept { +simdjson_inline simdjson_result value::operator[](const char *key) noexcept { return start_or_resume_object()[key]; } -simdjson_really_inline simdjson_result value::type() noexcept { +simdjson_inline simdjson_result value::type() noexcept { return iter.type(); } -simdjson_really_inline simdjson_result value::is_scalar() noexcept { +simdjson_inline simdjson_result value::is_scalar() noexcept { json_type this_type; auto error = type().get(this_type); if(error) { return error; } return ! ((this_type == json_type::array) || (this_type == json_type::object)); } -simdjson_really_inline bool value::is_negative() noexcept { +simdjson_inline bool value::is_negative() noexcept { return iter.is_negative(); } -simdjson_really_inline simdjson_result value::is_integer() noexcept { +simdjson_inline simdjson_result value::is_integer() noexcept { return iter.is_integer(); } -simdjson_warn_unused simdjson_really_inline simdjson_result value::get_number_type() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value::get_number_type() noexcept { return iter.get_number_type(); } -simdjson_warn_unused simdjson_really_inline simdjson_result value::get_number() noexcept { +simdjson_warn_unused simdjson_inline simdjson_result value::get_number() noexcept { return iter.get_number(); } -simdjson_really_inline std::string_view value::raw_json_token() noexcept { +simdjson_inline std::string_view value::raw_json_token() noexcept { return std::string_view(reinterpret_cast(iter.peek_start()), iter.peek_start_length()); } -simdjson_really_inline simdjson_result value::current_location() noexcept { +simdjson_inline simdjson_result value::current_location() noexcept { return iter.json_iter().current_location(); } -simdjson_really_inline int32_t value::current_depth() const noexcept{ +simdjson_inline int32_t value::current_depth() const noexcept{ return iter.json_iter().depth(); } -simdjson_really_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { +simdjson_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { json_type t; SIMDJSON_TRY(type().get(t)); switch (t) @@ -30814,7 +30243,7 @@ simdjson_really_inline simdjson_result value::at_pointer(std::string_view namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value &&value ) noexcept : implementation_simdjson_result_base( @@ -30822,203 +30251,203 @@ simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( error_code error ) noexcept : implementation_simdjson_result_base(error) { } -simdjson_really_inline simdjson_result simdjson_result::count_elements() & noexcept { +simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { if (error()) { return error(); } return first.count_elements(); } -simdjson_really_inline simdjson_result simdjson_result::count_fields() & noexcept { +simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { if (error()) { return error(); } return first.count_fields(); } -simdjson_really_inline simdjson_result simdjson_result::at(size_t index) noexcept { +simdjson_inline simdjson_result simdjson_result::at(size_t index) noexcept { if (error()) { return error(); } return first.at(index); } -simdjson_really_inline simdjson_result simdjson_result::begin() & noexcept { +simdjson_inline simdjson_result simdjson_result::begin() & noexcept { if (error()) { return error(); } return first.begin(); } -simdjson_really_inline simdjson_result simdjson_result::end() & noexcept { +simdjson_inline simdjson_result simdjson_result::end() & noexcept { if (error()) { return error(); } return {}; } -simdjson_really_inline simdjson_result simdjson_result::find_field(std::string_view key) noexcept { +simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) noexcept { if (error()) { return error(); } return first.find_field(key); } -simdjson_really_inline simdjson_result simdjson_result::find_field(const char *key) noexcept { +simdjson_inline simdjson_result simdjson_result::find_field(const char *key) noexcept { if (error()) { return error(); } return first.find_field(key); } -simdjson_really_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) noexcept { +simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) noexcept { if (error()) { return error(); } return first.find_field_unordered(key); } -simdjson_really_inline simdjson_result simdjson_result::find_field_unordered(const char *key) noexcept { +simdjson_inline simdjson_result simdjson_result::find_field_unordered(const char *key) noexcept { if (error()) { return error(); } return first.find_field_unordered(key); } -simdjson_really_inline simdjson_result simdjson_result::operator[](std::string_view key) noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) noexcept { if (error()) { return error(); } return first[key]; } -simdjson_really_inline simdjson_result simdjson_result::operator[](const char *key) noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](const char *key) noexcept { if (error()) { return error(); } return first[key]; } -simdjson_really_inline simdjson_result simdjson_result::get_array() noexcept { +simdjson_inline simdjson_result simdjson_result::get_array() noexcept { if (error()) { return error(); } return first.get_array(); } -simdjson_really_inline simdjson_result simdjson_result::get_object() noexcept { +simdjson_inline simdjson_result simdjson_result::get_object() noexcept { if (error()) { return error(); } return first.get_object(); } -simdjson_really_inline simdjson_result simdjson_result::get_uint64() noexcept { +simdjson_inline simdjson_result simdjson_result::get_uint64() noexcept { if (error()) { return error(); } return first.get_uint64(); } -simdjson_really_inline simdjson_result simdjson_result::get_uint64_in_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_uint64_in_string() noexcept { if (error()) { return error(); } return first.get_uint64_in_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_int64() noexcept { +simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { if (error()) { return error(); } return first.get_int64(); } -simdjson_really_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { if (error()) { return error(); } return first.get_int64_in_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_double() noexcept { +simdjson_inline simdjson_result simdjson_result::get_double() noexcept { if (error()) { return error(); } return first.get_double(); } -simdjson_really_inline simdjson_result simdjson_result::get_double_in_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_double_in_string() noexcept { if (error()) { return error(); } return first.get_double_in_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_string() noexcept { if (error()) { return error(); } return first.get_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { +simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { if (error()) { return error(); } return first.get_raw_json_string(); } -simdjson_really_inline simdjson_result simdjson_result::get_bool() noexcept { +simdjson_inline simdjson_result simdjson_result::get_bool() noexcept { if (error()) { return error(); } return first.get_bool(); } -simdjson_really_inline bool simdjson_result::is_null() noexcept { - if (error()) { return false; } +simdjson_inline simdjson_result simdjson_result::is_null() noexcept { + if (error()) { return error(); } return first.is_null(); } -template simdjson_really_inline simdjson_result simdjson_result::get() noexcept { +template simdjson_inline simdjson_result simdjson_result::get() noexcept { if (error()) { return error(); } return first.get(); } -template simdjson_really_inline error_code simdjson_result::get(T &out) noexcept { +template simdjson_inline error_code simdjson_result::get(T &out) noexcept { if (error()) { return error(); } return first.get(out); } -template<> simdjson_really_inline simdjson_result simdjson_result::get() noexcept { +template<> simdjson_inline simdjson_result simdjson_result::get() noexcept { if (error()) { return error(); } return std::move(first); } -template<> simdjson_really_inline error_code simdjson_result::get(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value &out) noexcept { +template<> simdjson_inline error_code simdjson_result::get(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value &out) noexcept { if (error()) { return error(); } out = first; return SUCCESS; } -simdjson_really_inline simdjson_result simdjson_result::type() noexcept { +simdjson_inline simdjson_result simdjson_result::type() noexcept { if (error()) { return error(); } return first.type(); } -simdjson_really_inline simdjson_result simdjson_result::is_scalar() noexcept { +simdjson_inline simdjson_result simdjson_result::is_scalar() noexcept { if (error()) { return error(); } return first.is_scalar(); } -simdjson_really_inline simdjson_result simdjson_result::is_negative() noexcept { +simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); } -simdjson_really_inline simdjson_result simdjson_result::is_integer() noexcept { +simdjson_inline simdjson_result simdjson_result::is_integer() noexcept { if (error()) { return error(); } return first.is_integer(); } -simdjson_really_inline simdjson_result simdjson_result::get_number_type() noexcept { +simdjson_inline simdjson_result simdjson_result::get_number_type() noexcept { if (error()) { return error(); } return first.get_number_type(); } -simdjson_really_inline simdjson_result simdjson_result::get_number() noexcept { +simdjson_inline simdjson_result simdjson_result::get_number() noexcept { if (error()) { return error(); } return first.get_number(); } #if SIMDJSON_EXCEPTIONS -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator uint64_t() noexcept(false) { +simdjson_inline simdjson_result::operator uint64_t() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator int64_t() noexcept(false) { +simdjson_inline simdjson_result::operator int64_t() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator double() noexcept(false) { +simdjson_inline simdjson_result::operator double() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator std::string_view() noexcept(false) { +simdjson_inline simdjson_result::operator std::string_view() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false) { +simdjson_inline simdjson_result::operator SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::raw_json_string() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } -simdjson_really_inline simdjson_result::operator bool() noexcept(false) { +simdjson_inline simdjson_result::operator bool() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; } #endif -simdjson_really_inline simdjson_result simdjson_result::raw_json_token() noexcept { +simdjson_inline simdjson_result simdjson_result::raw_json_token() noexcept { if (error()) { return error(); } return first.raw_json_token(); } -simdjson_really_inline simdjson_result simdjson_result::current_location() noexcept { +simdjson_inline simdjson_result simdjson_result::current_location() noexcept { if (error()) { return error(); } return first.current_location(); } -simdjson_really_inline int32_t simdjson_result::current_depth() const noexcept { +simdjson_inline simdjson_result simdjson_result::current_depth() const noexcept { if (error()) { return error(); } return first.current_depth(); } -simdjson_really_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { +simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { if (error()) { return error(); } return first.at_pointer(json_pointer); } @@ -31031,41 +30460,41 @@ namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { // clang 6 doesn't think the default constructor can be noexcept, so we make it explicit -simdjson_really_inline field::field() noexcept : std::pair() {} +simdjson_inline field::field() noexcept : std::pair() {} -simdjson_really_inline field::field(raw_json_string key, ondemand::value &&value) noexcept +simdjson_inline field::field(raw_json_string key, ondemand::value &&value) noexcept : std::pair(key, std::forward(value)) { } -simdjson_really_inline simdjson_result field::start(value_iterator &parent_iter) noexcept { +simdjson_inline simdjson_result field::start(value_iterator &parent_iter) noexcept { raw_json_string key; SIMDJSON_TRY( parent_iter.field_key().get(key) ); SIMDJSON_TRY( parent_iter.field_value() ); return field::start(parent_iter, key); } -simdjson_really_inline simdjson_result field::start(const value_iterator &parent_iter, raw_json_string key) noexcept { +simdjson_inline simdjson_result field::start(const value_iterator &parent_iter, raw_json_string key) noexcept { return field(key, parent_iter.child()); } -simdjson_really_inline simdjson_warn_unused simdjson_result field::unescaped_key() noexcept { +simdjson_inline simdjson_warn_unused simdjson_result field::unescaped_key() noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us. - simdjson_result answer = first.unescape(second.iter.string_buf_loc()); + simdjson_result answer = first.unescape(second.iter.json_iter()); first.consume(); return answer; } -simdjson_really_inline raw_json_string field::key() const noexcept { +simdjson_inline raw_json_string field::key() const noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. return first; } -simdjson_really_inline value &field::value() & noexcept { +simdjson_inline value &field::value() & noexcept { return second; } -simdjson_really_inline value field::value() && noexcept { +simdjson_inline value field::value() && noexcept { return std::forward(*this).second; } @@ -31075,7 +30504,7 @@ simdjson_really_inline value field::value() && noexcept { namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::field &&value ) noexcept : implementation_simdjson_result_base( @@ -31083,22 +30512,22 @@ simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( error_code error ) noexcept : implementation_simdjson_result_base(error) { } -simdjson_really_inline simdjson_result simdjson_result::key() noexcept { +simdjson_inline simdjson_result simdjson_result::key() noexcept { if (error()) { return error(); } return first.key(); } -simdjson_really_inline simdjson_result simdjson_result::unescaped_key() noexcept { +simdjson_inline simdjson_result simdjson_result::unescaped_key() noexcept { if (error()) { return error(); } return first.unescaped_key(); } -simdjson_really_inline simdjson_result simdjson_result::value() noexcept { +simdjson_inline simdjson_result simdjson_result::value() noexcept { if (error()) { return error(); } return std::move(first.value()); } @@ -31110,46 +30539,46 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline simdjson_result object::find_field_unordered(const std::string_view key) & noexcept { +simdjson_inline simdjson_result object::find_field_unordered(const std::string_view key) & noexcept { bool has_value; SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) ); if (!has_value) { return NO_SUCH_FIELD; } return value(iter.child()); } -simdjson_really_inline simdjson_result object::find_field_unordered(const std::string_view key) && noexcept { +simdjson_inline simdjson_result object::find_field_unordered(const std::string_view key) && noexcept { bool has_value; SIMDJSON_TRY( iter.find_field_unordered_raw(key).get(has_value) ); if (!has_value) { return NO_SUCH_FIELD; } return value(iter.child()); } -simdjson_really_inline simdjson_result object::operator[](const std::string_view key) & noexcept { +simdjson_inline simdjson_result object::operator[](const std::string_view key) & noexcept { return find_field_unordered(key); } -simdjson_really_inline simdjson_result object::operator[](const std::string_view key) && noexcept { +simdjson_inline simdjson_result object::operator[](const std::string_view key) && noexcept { return std::forward(*this).find_field_unordered(key); } -simdjson_really_inline simdjson_result object::find_field(const std::string_view key) & noexcept { +simdjson_inline simdjson_result object::find_field(const std::string_view key) & noexcept { bool has_value; SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) ); if (!has_value) { return NO_SUCH_FIELD; } return value(iter.child()); } -simdjson_really_inline simdjson_result object::find_field(const std::string_view key) && noexcept { +simdjson_inline simdjson_result object::find_field(const std::string_view key) && noexcept { bool has_value; SIMDJSON_TRY( iter.find_field_raw(key).get(has_value) ); if (!has_value) { return NO_SUCH_FIELD; } return value(iter.child()); } -simdjson_really_inline simdjson_result object::start(value_iterator &iter) noexcept { +simdjson_inline simdjson_result object::start(value_iterator &iter) noexcept { SIMDJSON_TRY( iter.start_object().error() ); return object(iter); } -simdjson_really_inline simdjson_result object::start_root(value_iterator &iter) noexcept { +simdjson_inline simdjson_result object::start_root(value_iterator &iter) noexcept { SIMDJSON_TRY( iter.start_root_object().error() ); return object(iter); } -simdjson_really_inline error_code object::consume() noexcept { +simdjson_inline error_code object::consume() noexcept { if(iter.is_at_key()) { /** * whenever you are pointing at a key, calling skip_child() is @@ -31175,7 +30604,7 @@ simdjson_really_inline error_code object::consume() noexcept { return error_skip; } -simdjson_really_inline simdjson_result object::raw_json() noexcept { +simdjson_inline simdjson_result object::raw_json() noexcept { const uint8_t * starting_point{iter.peek_start()}; auto error = consume(); if(error) { return error; } @@ -31183,27 +30612,27 @@ simdjson_really_inline simdjson_result object::raw_json() noex return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } -simdjson_really_inline simdjson_result object::started(value_iterator &iter) noexcept { +simdjson_inline simdjson_result object::started(value_iterator &iter) noexcept { SIMDJSON_TRY( iter.started_object().error() ); return object(iter); } -simdjson_really_inline object object::resume(const value_iterator &iter) noexcept { +simdjson_inline object object::resume(const value_iterator &iter) noexcept { return iter; } -simdjson_really_inline object::object(const value_iterator &_iter) noexcept +simdjson_inline object::object(const value_iterator &_iter) noexcept : iter{_iter} { } -simdjson_really_inline simdjson_result object::begin() noexcept { -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +simdjson_inline simdjson_result object::begin() noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS if (!iter.is_at_iterator_start()) { return OUT_OF_ORDER_ITERATION; } #endif return object_iterator(iter); } -simdjson_really_inline simdjson_result object::end() noexcept { +simdjson_inline simdjson_result object::end() noexcept { return object_iterator(iter); } @@ -31247,7 +30676,7 @@ inline simdjson_result object::at_pointer(std::string_view json_pointer) return child; } -simdjson_really_inline simdjson_result object::count_fields() & noexcept { +simdjson_inline simdjson_result object::count_fields() & noexcept { size_t count{0}; // Important: we do not consume any of the values. for(simdjson_unused auto v : *this) { count++; } @@ -31259,14 +30688,14 @@ simdjson_really_inline simdjson_result object::count_fields() & noexcept return count; } -simdjson_really_inline simdjson_result object::is_empty() & noexcept { +simdjson_inline simdjson_result object::is_empty() & noexcept { bool is_not_empty; auto error = iter.reset_object().get(is_not_empty); if(error) { return error; } return !is_not_empty; } -simdjson_really_inline simdjson_result object::reset() & noexcept { +simdjson_inline simdjson_result object::reset() & noexcept { return iter.reset_object(); } @@ -31276,45 +30705,45 @@ simdjson_really_inline simdjson_result object::reset() & noexcept { namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object &&value) noexcept : implementation_simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : implementation_simdjson_result_base(error) {} -simdjson_really_inline simdjson_result simdjson_result::begin() noexcept { +simdjson_inline simdjson_result simdjson_result::begin() noexcept { if (error()) { return error(); } return first.begin(); } -simdjson_really_inline simdjson_result simdjson_result::end() noexcept { +simdjson_inline simdjson_result simdjson_result::end() noexcept { if (error()) { return error(); } return first.end(); } -simdjson_really_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { if (error()) { return error(); } return first.find_field_unordered(key); } -simdjson_really_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) && noexcept { +simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) && noexcept { if (error()) { return error(); } return std::forward(first).find_field_unordered(key); } -simdjson_really_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { if (error()) { return error(); } return first[key]; } -simdjson_really_inline simdjson_result simdjson_result::operator[](std::string_view key) && noexcept { +simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) && noexcept { if (error()) { return error(); } return std::forward(first)[key]; } -simdjson_really_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { +simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { if (error()) { return error(); } return first.find_field(key); } -simdjson_really_inline simdjson_result simdjson_result::find_field(std::string_view key) && noexcept { +simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) && noexcept { if (error()) { return error(); } return std::forward(first).find_field(key); } -simdjson_really_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { +simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { if (error()) { return error(); } return first.at_pointer(json_pointer); } @@ -31329,7 +30758,7 @@ inline simdjson_result simdjson_result simdjson_result::count_fields() & noexcept { +simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { if (error()) { return error(); } return first.count_fields(); } @@ -31341,11 +30770,11 @@ namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand { -simdjson_really_inline parser::parser(size_t max_capacity) noexcept +simdjson_inline parser::parser(size_t max_capacity) noexcept : _max_capacity{max_capacity} { } -simdjson_warn_unused simdjson_really_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { +simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { if (new_capacity > max_capacity()) { return CAPACITY; } if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } @@ -31353,7 +30782,7 @@ simdjson_warn_unused simdjson_really_inline error_code parser::allocate(size_t n _capacity = 0; size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * new_capacity / 3 + SIMDJSON_PADDING, 64); string_buf.reset(new (std::nothrow) uint8_t[string_capacity]); -#ifdef SIMDJSON_DEVELOPMENT_CHECKS +#if SIMDJSON_DEVELOPMENT_CHECKS start_positions.reset(new (std::nothrow) token_position[new_max_depth]); #endif if (implementation) { @@ -31367,7 +30796,7 @@ simdjson_warn_unused simdjson_really_inline error_code parser::allocate(size_t n return SUCCESS; } -simdjson_warn_unused simdjson_really_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { +simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } // Allocate if needed @@ -31380,37 +30809,37 @@ simdjson_warn_unused simdjson_really_inline simdjson_result parser::it return document::start({ reinterpret_cast(json.data()), this }); } -simdjson_warn_unused simdjson_really_inline simdjson_result parser::iterate(const char *json, size_t len, size_t allocated) & noexcept { +simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const char *json, size_t len, size_t allocated) & noexcept { return iterate(padded_string_view(json, len, allocated)); } -simdjson_warn_unused simdjson_really_inline simdjson_result parser::iterate(const uint8_t *json, size_t len, size_t allocated) & noexcept { +simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const uint8_t *json, size_t len, size_t allocated) & noexcept { return iterate(padded_string_view(json, len, allocated)); } -simdjson_warn_unused simdjson_really_inline simdjson_result parser::iterate(std::string_view json, size_t allocated) & noexcept { +simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(std::string_view json, size_t allocated) & noexcept { return iterate(padded_string_view(json, allocated)); } -simdjson_warn_unused simdjson_really_inline simdjson_result parser::iterate(const std::string &json) & noexcept { +simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const std::string &json) & noexcept { return iterate(padded_string_view(json)); } -simdjson_warn_unused simdjson_really_inline simdjson_result parser::iterate(const simdjson_result &result) & noexcept { +simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const simdjson_result &result) & noexcept { // We don't presently have a way to temporarily get a const T& from a simdjson_result without throwing an exception SIMDJSON_TRY( result.error() ); padded_string_view json = result.value_unsafe(); return iterate(json); } -simdjson_warn_unused simdjson_really_inline simdjson_result parser::iterate(const simdjson_result &result) & noexcept { +simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const simdjson_result &result) & noexcept { // We don't presently have a way to temporarily get a const T& from a simdjson_result without throwing an exception SIMDJSON_TRY( result.error() ); const padded_string &json = result.value_unsafe(); return iterate(json); } -simdjson_warn_unused simdjson_really_inline simdjson_result parser::iterate_raw(padded_string_view json) & noexcept { +simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_raw(padded_string_view json) & noexcept { if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } // Allocate if needed @@ -31437,34 +30866,41 @@ inline simdjson_result parser::iterate_many(const padded_string return iterate_many(s.data(), s.length(), batch_size); } -simdjson_really_inline size_t parser::capacity() const noexcept { +simdjson_inline size_t parser::capacity() const noexcept { return _capacity; } -simdjson_really_inline size_t parser::max_capacity() const noexcept { +simdjson_inline size_t parser::max_capacity() const noexcept { return _max_capacity; } -simdjson_really_inline size_t parser::max_depth() const noexcept { +simdjson_inline size_t parser::max_depth() const noexcept { return _max_depth; } -simdjson_really_inline void parser::set_max_capacity(size_t max_capacity) noexcept { - size_t MINIMAL_DOCUMENT_CAPACITY = 32; - if(max_capacity < MINIMAL_DOCUMENT_CAPACITY) { +simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { + if(max_capacity < dom::MINIMAL_DOCUMENT_CAPACITY) { _max_capacity = max_capacity; } else { - _max_capacity = MINIMAL_DOCUMENT_CAPACITY; + _max_capacity = dom::MINIMAL_DOCUMENT_CAPACITY; } } +simdjson_inline simdjson_warn_unused simdjson_result parser::unescape(raw_json_string in, uint8_t *&dst) const noexcept { + uint8_t *end = implementation->parse_string(in.buf, dst); + if (!end) { return STRING_ERROR; } + std::string_view result(reinterpret_cast(dst), end-dst); + dst = end; + return result; +} + } // namespace ondemand } // namespace SIMDJSON_BUILTIN_IMPLEMENTATION } // namespace simdjson namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::parser &&value) noexcept +simdjson_inline simdjson_result::simdjson_result(SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::parser &&value) noexcept : implementation_simdjson_result_base(std::forward(value)) {} -simdjson_really_inline simdjson_result::simdjson_result(error_code error) noexcept +simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept : implementation_simdjson_result_base(error) {} } // namespace simdjson @@ -31552,7 +30988,7 @@ inline void stage1_worker::run(document_stream * ds, parser * stage1, size_t nex #endif // SIMDJSON_THREADS_ENABLED -simdjson_really_inline document_stream::document_stream( +simdjson_inline document_stream::document_stream( ondemand::parser &_parser, const uint8_t *_buf, size_t _len, @@ -31574,7 +31010,7 @@ simdjson_really_inline document_stream::document_stream( #endif } -simdjson_really_inline document_stream::document_stream() noexcept +simdjson_inline document_stream::document_stream() noexcept : parser{nullptr}, buf{nullptr}, len{0}, @@ -31586,7 +31022,7 @@ simdjson_really_inline document_stream::document_stream() noexcept { } -simdjson_really_inline document_stream::~document_stream() noexcept +simdjson_inline document_stream::~document_stream() noexcept { #ifdef SIMDJSON_THREADS_ENABLED worker.reset(); @@ -31602,20 +31038,20 @@ inline size_t document_stream::truncated_bytes() const noexcept { return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1]; } -simdjson_really_inline document_stream::iterator::iterator() noexcept +simdjson_inline document_stream::iterator::iterator() noexcept : stream{nullptr}, finished{true} { } -simdjson_really_inline document_stream::iterator::iterator(document_stream* _stream, bool is_end) noexcept +simdjson_inline document_stream::iterator::iterator(document_stream* _stream, bool is_end) noexcept : stream{_stream}, finished{is_end} { } -simdjson_really_inline simdjson_result document_stream::iterator::operator*() noexcept { +simdjson_inline simdjson_result document_stream::iterator::operator*() noexcept { //if(stream->error) { return stream->error; } return simdjson_result(stream->doc, stream->error); } -simdjson_really_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { +simdjson_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { // If there is an error, then we want the iterator // to be finished, no matter what. (E.g., we do not // keep generating documents with errors, or go beyond @@ -31642,17 +31078,17 @@ simdjson_really_inline document_stream::iterator& document_stream::iterator::ope return *this; } -simdjson_really_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { +simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } -simdjson_really_inline document_stream::iterator document_stream::begin() noexcept { +simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. return iterator(this, error == EMPTY); } -simdjson_really_inline document_stream::iterator document_stream::end() noexcept { +simdjson_inline document_stream::iterator document_stream::end() noexcept { return iterator(this, true); } @@ -31782,11 +31218,11 @@ inline error_code document_stream::run_stage1(ondemand::parser &p, size_t _batch } } -simdjson_really_inline size_t document_stream::iterator::current_index() const noexcept { +simdjson_inline size_t document_stream::iterator::current_index() const noexcept { return stream->doc_index; } -simdjson_really_inline std::string_view document_stream::iterator::source() const noexcept { +simdjson_inline std::string_view document_stream::iterator::source() const noexcept { auto depth = stream->doc.iter.depth(); auto cur_struct_index = stream->doc.iter._root - stream->parser->implementation->structural_indexes.get(); @@ -31861,13 +31297,13 @@ inline void document_stream::start_stage1_thread() noexcept { namespace simdjson { -simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( error_code error ) noexcept : implementation_simdjson_result_base(error) { } -simdjson_really_inline simdjson_result::simdjson_result( +simdjson_inline simdjson_result::simdjson_result( SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::document_stream &&value ) noexcept : implementation_simdjson_result_base( diff --git a/tests/conftest.py b/tests/conftest.py index a3d14a0..c598996 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,8 @@ def doc(parser): "int64": -1, "uint64": 18446744073709551615, "double": 1.1, + "double_zero": 0.0, + "double_whole": 1.0, "string": "test", "bool": true, "null_value": null diff --git a/tests/test_float.py b/tests/test_float.py new file mode 100644 index 0000000..ca5e427 --- /dev/null +++ b/tests/test_float.py @@ -0,0 +1,9 @@ + +def test_parser_understands_float(parser): + doc = parser.parse('1.0') + assert type(doc) is float + + +def test_mini_does_not_drop_zero(parser): + doc = parser.parse(b'[0.0, 0.5, 1.0]') + assert doc.mini == b'[0.0,0.5,1.0]'