1/* Implementation helper: a struct that looks like a tuple. 2 See timemodule and posixmodule for example uses. 3 4 The structseq helper is considered an internal CPython implementation 5 detail. Docs for modules using structseqs should call them 6 "named tuples" (be sure to include a space between the two 7 words and add a link back to the term in Docs/glossary.rst). 8*/ 9 10#include "Python.h" 11#include "pycore_tuple.h" // _PyTuple_FromArray() 12#include "pycore_object.h" // _PyObject_GC_TRACK() 13#include "structmember.h" // PyMemberDef 14#include "pycore_structseq.h" // PyStructSequence_InitType() 15#include "pycore_initconfig.h" // _PyStatus_OK() 16 17static const char visible_length_key[] = "n_sequence_fields"; 18static const char real_length_key[] = "n_fields"; 19static const char unnamed_fields_key[] = "n_unnamed_fields"; 20static const char match_args_key[] = "__match_args__"; 21 22/* Fields with this name have only a field index, not a field name. 23 They are only allowed for indices < n_visible_fields. */ 24const char * const PyStructSequence_UnnamedField = "unnamed field"; 25 26static Py_ssize_t 27get_type_attr_as_size(PyTypeObject *tp, PyObject *name) 28{ 29 PyObject *v = PyDict_GetItemWithError(tp->tp_dict, name); 30 if (v == NULL && !PyErr_Occurred()) { 31 PyErr_Format(PyExc_TypeError, 32 "Missed attribute '%U' of type %s", 33 name, tp->tp_name); 34 } 35 return PyLong_AsSsize_t(v); 36} 37 38#define VISIBLE_SIZE(op) Py_SIZE(op) 39#define VISIBLE_SIZE_TP(tp) \ 40 get_type_attr_as_size(tp, &_Py_ID(n_sequence_fields)) 41#define REAL_SIZE_TP(tp) \ 42 get_type_attr_as_size(tp, &_Py_ID(n_fields)) 43#define REAL_SIZE(op) REAL_SIZE_TP(Py_TYPE(op)) 44 45#define UNNAMED_FIELDS_TP(tp) \ 46 get_type_attr_as_size(tp, &_Py_ID(n_unnamed_fields)) 47#define UNNAMED_FIELDS(op) UNNAMED_FIELDS_TP(Py_TYPE(op)) 48 49 50PyObject * 51PyStructSequence_New(PyTypeObject *type) 52{ 53 PyStructSequence *obj; 54 Py_ssize_t size = REAL_SIZE_TP(type), i; 55 if (size < 0) { 56 return NULL; 57 } 58 Py_ssize_t vsize = VISIBLE_SIZE_TP(type); 59 if (vsize < 0) { 60 return NULL; 61 } 62 63 obj = PyObject_GC_NewVar(PyStructSequence, type, size); 64 if (obj == NULL) 65 return NULL; 66 /* Hack the size of the variable object, so invisible fields don't appear 67 to Python code. */ 68 Py_SET_SIZE(obj, vsize); 69 for (i = 0; i < size; i++) 70 obj->ob_item[i] = NULL; 71 72 return (PyObject*)obj; 73} 74 75void 76PyStructSequence_SetItem(PyObject* op, Py_ssize_t i, PyObject* v) 77{ 78 PyStructSequence_SET_ITEM(op, i, v); 79} 80 81PyObject* 82PyStructSequence_GetItem(PyObject* op, Py_ssize_t i) 83{ 84 return PyStructSequence_GET_ITEM(op, i); 85} 86 87 88static int 89structseq_traverse(PyStructSequence *obj, visitproc visit, void *arg) 90{ 91 if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HEAPTYPE) { 92 Py_VISIT(Py_TYPE(obj)); 93 } 94 Py_ssize_t i, size; 95 size = REAL_SIZE(obj); 96 for (i = 0; i < size; ++i) { 97 Py_VISIT(obj->ob_item[i]); 98 } 99 return 0; 100} 101 102static void 103structseq_dealloc(PyStructSequence *obj) 104{ 105 Py_ssize_t i, size; 106 PyObject_GC_UnTrack(obj); 107 108 PyTypeObject *tp = Py_TYPE(obj); 109 size = REAL_SIZE(obj); 110 for (i = 0; i < size; ++i) { 111 Py_XDECREF(obj->ob_item[i]); 112 } 113 PyObject_GC_Del(obj); 114 if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { 115 Py_DECREF(tp); 116 } 117} 118 119/*[clinic input] 120class structseq "PyStructSequence *" "NULL" 121[clinic start generated code]*/ 122/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9d781c6922c77752]*/ 123 124#include "clinic/structseq.c.h" 125 126/*[clinic input] 127@classmethod 128structseq.__new__ as structseq_new 129 sequence as arg: object 130 dict: object(c_default="NULL") = {} 131[clinic start generated code]*/ 132 133static PyObject * 134structseq_new_impl(PyTypeObject *type, PyObject *arg, PyObject *dict) 135/*[clinic end generated code: output=baa082e788b171da input=90532511101aa3fb]*/ 136{ 137 PyObject *ob; 138 PyStructSequence *res = NULL; 139 Py_ssize_t len, min_len, max_len, i, n_unnamed_fields; 140 141 min_len = VISIBLE_SIZE_TP(type); 142 if (min_len < 0) { 143 return NULL; 144 } 145 max_len = REAL_SIZE_TP(type); 146 if (max_len < 0) { 147 return NULL; 148 } 149 n_unnamed_fields = UNNAMED_FIELDS_TP(type); 150 if (n_unnamed_fields < 0) { 151 return NULL; 152 } 153 154 arg = PySequence_Fast(arg, "constructor requires a sequence"); 155 156 if (!arg) { 157 return NULL; 158 } 159 160 if (dict && !PyDict_Check(dict)) { 161 PyErr_Format(PyExc_TypeError, 162 "%.500s() takes a dict as second arg, if any", 163 type->tp_name); 164 Py_DECREF(arg); 165 return NULL; 166 } 167 168 len = PySequence_Fast_GET_SIZE(arg); 169 if (min_len != max_len) { 170 if (len < min_len) { 171 PyErr_Format(PyExc_TypeError, 172 "%.500s() takes an at least %zd-sequence (%zd-sequence given)", 173 type->tp_name, min_len, len); 174 Py_DECREF(arg); 175 return NULL; 176 } 177 178 if (len > max_len) { 179 PyErr_Format(PyExc_TypeError, 180 "%.500s() takes an at most %zd-sequence (%zd-sequence given)", 181 type->tp_name, max_len, len); 182 Py_DECREF(arg); 183 return NULL; 184 } 185 } 186 else { 187 if (len != min_len) { 188 PyErr_Format(PyExc_TypeError, 189 "%.500s() takes a %zd-sequence (%zd-sequence given)", 190 type->tp_name, min_len, len); 191 Py_DECREF(arg); 192 return NULL; 193 } 194 } 195 196 res = (PyStructSequence*) PyStructSequence_New(type); 197 if (res == NULL) { 198 Py_DECREF(arg); 199 return NULL; 200 } 201 for (i = 0; i < len; ++i) { 202 PyObject *v = PySequence_Fast_GET_ITEM(arg, i); 203 Py_INCREF(v); 204 res->ob_item[i] = v; 205 } 206 Py_DECREF(arg); 207 for (; i < max_len; ++i) { 208 if (dict == NULL) { 209 ob = Py_None; 210 } 211 else { 212 ob = _PyDict_GetItemStringWithError(dict, 213 type->tp_members[i-n_unnamed_fields].name); 214 if (ob == NULL) { 215 if (PyErr_Occurred()) { 216 Py_DECREF(res); 217 return NULL; 218 } 219 ob = Py_None; 220 } 221 } 222 Py_INCREF(ob); 223 res->ob_item[i] = ob; 224 } 225 226 _PyObject_GC_TRACK(res); 227 return (PyObject*) res; 228} 229 230 231static PyObject * 232structseq_repr(PyStructSequence *obj) 233{ 234 PyTypeObject *typ = Py_TYPE(obj); 235 _PyUnicodeWriter writer; 236 237 /* Write "typename(" */ 238 PyObject *type_name = PyUnicode_DecodeUTF8(typ->tp_name, 239 strlen(typ->tp_name), 240 NULL); 241 if (type_name == NULL) { 242 return NULL; 243 } 244 245 _PyUnicodeWriter_Init(&writer); 246 writer.overallocate = 1; 247 /* count 5 characters per item: "x=1, " */ 248 writer.min_length = (PyUnicode_GET_LENGTH(type_name) + 1 249 + VISIBLE_SIZE(obj) * 5 + 1); 250 251 if (_PyUnicodeWriter_WriteStr(&writer, type_name) < 0) { 252 Py_DECREF(type_name); 253 goto error; 254 } 255 Py_DECREF(type_name); 256 257 if (_PyUnicodeWriter_WriteChar(&writer, '(') < 0) { 258 goto error; 259 } 260 261 for (Py_ssize_t i=0; i < VISIBLE_SIZE(obj); i++) { 262 if (i > 0) { 263 /* Write ", " */ 264 if (_PyUnicodeWriter_WriteASCIIString(&writer, ", ", 2) < 0) { 265 goto error; 266 } 267 } 268 269 /* Write "name=repr" */ 270 const char *name_utf8 = typ->tp_members[i].name; 271 if (name_utf8 == NULL) { 272 PyErr_Format(PyExc_SystemError, "In structseq_repr(), member %zd name is NULL" 273 " for type %.500s", i, typ->tp_name); 274 goto error; 275 } 276 277 PyObject *name = PyUnicode_DecodeUTF8(name_utf8, strlen(name_utf8), NULL); 278 if (name == NULL) { 279 goto error; 280 } 281 if (_PyUnicodeWriter_WriteStr(&writer, name) < 0) { 282 Py_DECREF(name); 283 goto error; 284 } 285 Py_DECREF(name); 286 287 if (_PyUnicodeWriter_WriteChar(&writer, '=') < 0) { 288 goto error; 289 } 290 291 PyObject *value = PyStructSequence_GET_ITEM(obj, i); 292 assert(value != NULL); 293 PyObject *repr = PyObject_Repr(value); 294 if (repr == NULL) { 295 goto error; 296 } 297 if (_PyUnicodeWriter_WriteStr(&writer, repr) < 0) { 298 Py_DECREF(repr); 299 goto error; 300 } 301 Py_DECREF(repr); 302 } 303 304 if (_PyUnicodeWriter_WriteChar(&writer, ')') < 0) { 305 goto error; 306 } 307 308 return _PyUnicodeWriter_Finish(&writer); 309 310error: 311 _PyUnicodeWriter_Dealloc(&writer); 312 return NULL; 313} 314 315 316static PyObject * 317structseq_reduce(PyStructSequence* self, PyObject *Py_UNUSED(ignored)) 318{ 319 PyObject* tup = NULL; 320 PyObject* dict = NULL; 321 PyObject* result; 322 Py_ssize_t n_fields, n_visible_fields, n_unnamed_fields, i; 323 324 n_fields = REAL_SIZE(self); 325 if (n_fields < 0) { 326 return NULL; 327 } 328 n_visible_fields = VISIBLE_SIZE(self); 329 n_unnamed_fields = UNNAMED_FIELDS(self); 330 if (n_unnamed_fields < 0) { 331 return NULL; 332 } 333 tup = _PyTuple_FromArray(self->ob_item, n_visible_fields); 334 if (!tup) 335 goto error; 336 337 dict = PyDict_New(); 338 if (!dict) 339 goto error; 340 341 for (i = n_visible_fields; i < n_fields; i++) { 342 const char *n = Py_TYPE(self)->tp_members[i-n_unnamed_fields].name; 343 if (PyDict_SetItemString(dict, n, self->ob_item[i]) < 0) 344 goto error; 345 } 346 347 result = Py_BuildValue("(O(OO))", Py_TYPE(self), tup, dict); 348 349 Py_DECREF(tup); 350 Py_DECREF(dict); 351 352 return result; 353 354error: 355 Py_XDECREF(tup); 356 Py_XDECREF(dict); 357 return NULL; 358} 359 360static PyMethodDef structseq_methods[] = { 361 {"__reduce__", (PyCFunction)structseq_reduce, METH_NOARGS, NULL}, 362 {NULL, NULL} 363}; 364 365static Py_ssize_t 366count_members(PyStructSequence_Desc *desc, Py_ssize_t *n_unnamed_members) { 367 Py_ssize_t i; 368 369 *n_unnamed_members = 0; 370 for (i = 0; desc->fields[i].name != NULL; ++i) { 371 if (desc->fields[i].name == PyStructSequence_UnnamedField) { 372 (*n_unnamed_members)++; 373 } 374 } 375 return i; 376} 377 378static int 379initialize_structseq_dict(PyStructSequence_Desc *desc, PyObject* dict, 380 Py_ssize_t n_members, Py_ssize_t n_unnamed_members) { 381 PyObject *v; 382 383#define SET_DICT_FROM_SIZE(key, value) \ 384 do { \ 385 v = PyLong_FromSsize_t(value); \ 386 if (v == NULL) { \ 387 return -1; \ 388 } \ 389 if (PyDict_SetItemString(dict, key, v) < 0) { \ 390 Py_DECREF(v); \ 391 return -1; \ 392 } \ 393 Py_DECREF(v); \ 394 } while (0) 395 396 SET_DICT_FROM_SIZE(visible_length_key, desc->n_in_sequence); 397 SET_DICT_FROM_SIZE(real_length_key, n_members); 398 SET_DICT_FROM_SIZE(unnamed_fields_key, n_unnamed_members); 399 400 // Prepare and set __match_args__ 401 Py_ssize_t i, k; 402 PyObject* keys = PyTuple_New(desc->n_in_sequence); 403 if (keys == NULL) { 404 return -1; 405 } 406 407 for (i = k = 0; i < desc->n_in_sequence; ++i) { 408 if (desc->fields[i].name == PyStructSequence_UnnamedField) { 409 continue; 410 } 411 PyObject* new_member = PyUnicode_FromString(desc->fields[i].name); 412 if (new_member == NULL) { 413 goto error; 414 } 415 PyTuple_SET_ITEM(keys, k, new_member); 416 k++; 417 } 418 419 if (_PyTuple_Resize(&keys, k) == -1) { 420 goto error; 421 } 422 423 if (PyDict_SetItemString(dict, match_args_key, keys) < 0) { 424 goto error; 425 } 426 427 Py_DECREF(keys); 428 return 0; 429 430error: 431 Py_DECREF(keys); 432 return -1; 433} 434 435static void 436initialize_members(PyStructSequence_Desc *desc, PyMemberDef* members, 437 Py_ssize_t n_members) { 438 Py_ssize_t i, k; 439 440 for (i = k = 0; i < n_members; ++i) { 441 if (desc->fields[i].name == PyStructSequence_UnnamedField) { 442 continue; 443 } 444 445 /* The names and docstrings in these MemberDefs are statically */ 446 /* allocated so it is expected that they'll outlive the MemberDef */ 447 members[k].name = desc->fields[i].name; 448 members[k].type = T_OBJECT; 449 members[k].offset = offsetof(PyStructSequence, ob_item) 450 + i * sizeof(PyObject*); 451 members[k].flags = READONLY; 452 members[k].doc = desc->fields[i].doc; 453 k++; 454 } 455 members[k].name = NULL; 456} 457 458 459int 460_PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc, 461 unsigned long tp_flags) 462{ 463 PyMemberDef *members; 464 Py_ssize_t n_members, n_unnamed_members; 465 466#ifdef Py_TRACE_REFS 467 /* if the type object was chained, unchain it first 468 before overwriting its storage */ 469 if (type->ob_base.ob_base._ob_next) { 470 _Py_ForgetReference((PyObject *)type); 471 } 472#endif 473 474 /* PyTypeObject has already been initialized */ 475 if (Py_REFCNT(type) != 0) { 476 PyErr_BadInternalCall(); 477 return -1; 478 } 479 480 type->tp_name = desc->name; 481 type->tp_basicsize = sizeof(PyStructSequence) - sizeof(PyObject *); 482 type->tp_itemsize = sizeof(PyObject *); 483 type->tp_dealloc = (destructor)structseq_dealloc; 484 type->tp_repr = (reprfunc)structseq_repr; 485 type->tp_doc = desc->doc; 486 type->tp_base = &PyTuple_Type; 487 type->tp_methods = structseq_methods; 488 type->tp_new = structseq_new; 489 type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | tp_flags; 490 type->tp_traverse = (traverseproc) structseq_traverse; 491 492 n_members = count_members(desc, &n_unnamed_members); 493 members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1); 494 if (members == NULL) { 495 PyErr_NoMemory(); 496 return -1; 497 } 498 initialize_members(desc, members, n_members); 499 type->tp_members = members; 500 501 if (PyType_Ready(type) < 0) { 502 PyMem_Free(members); 503 return -1; 504 } 505 Py_INCREF(type); 506 507 if (initialize_structseq_dict( 508 desc, type->tp_dict, n_members, n_unnamed_members) < 0) { 509 PyMem_Free(members); 510 Py_DECREF(type); 511 return -1; 512 } 513 514 return 0; 515} 516 517int 518PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) 519{ 520 return _PyStructSequence_InitType(type, desc, 0); 521} 522 523void 524PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) 525{ 526 (void)PyStructSequence_InitType2(type, desc); 527} 528 529 530void 531_PyStructSequence_FiniType(PyTypeObject *type) 532{ 533 // Ensure that the type is initialized 534 assert(type->tp_name != NULL); 535 assert(type->tp_base == &PyTuple_Type); 536 537 // Cannot delete a type if it still has subclasses 538 if (type->tp_subclasses != NULL) { 539 return; 540 } 541 542 // Undo PyStructSequence_NewType() 543 type->tp_name = NULL; 544 PyMem_Free(type->tp_members); 545 546 _PyStaticType_Dealloc(type); 547 assert(Py_REFCNT(type) == 1); 548 // Undo Py_INCREF(type) of _PyStructSequence_InitType(). 549 // Don't use Py_DECREF(): static type must not be deallocated 550 Py_SET_REFCNT(type, 0); 551#ifdef Py_REF_DEBUG 552 _Py_RefTotal--; 553#endif 554 555 // Make sure that _PyStructSequence_InitType() will initialize 556 // the type again 557 assert(Py_REFCNT(type) == 0); 558 assert(type->tp_name == NULL); 559} 560 561 562PyTypeObject * 563_PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags) 564{ 565 PyMemberDef *members; 566 PyTypeObject *type; 567 PyType_Slot slots[8]; 568 PyType_Spec spec; 569 Py_ssize_t n_members, n_unnamed_members; 570 571 /* Initialize MemberDefs */ 572 n_members = count_members(desc, &n_unnamed_members); 573 members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1); 574 if (members == NULL) { 575 PyErr_NoMemory(); 576 return NULL; 577 } 578 initialize_members(desc, members, n_members); 579 580 /* Initialize Slots */ 581 slots[0] = (PyType_Slot){Py_tp_dealloc, (destructor)structseq_dealloc}; 582 slots[1] = (PyType_Slot){Py_tp_repr, (reprfunc)structseq_repr}; 583 slots[2] = (PyType_Slot){Py_tp_doc, (void *)desc->doc}; 584 slots[3] = (PyType_Slot){Py_tp_methods, structseq_methods}; 585 slots[4] = (PyType_Slot){Py_tp_new, structseq_new}; 586 slots[5] = (PyType_Slot){Py_tp_members, members}; 587 slots[6] = (PyType_Slot){Py_tp_traverse, (traverseproc)structseq_traverse}; 588 slots[7] = (PyType_Slot){0, 0}; 589 590 /* Initialize Spec */ 591 /* The name in this PyType_Spec is statically allocated so it is */ 592 /* expected that it'll outlive the PyType_Spec */ 593 spec.name = desc->name; 594 spec.basicsize = sizeof(PyStructSequence) - sizeof(PyObject *); 595 spec.itemsize = sizeof(PyObject *); 596 spec.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | tp_flags; 597 spec.slots = slots; 598 599 type = (PyTypeObject *)PyType_FromSpecWithBases(&spec, (PyObject *)&PyTuple_Type); 600 PyMem_Free(members); 601 if (type == NULL) { 602 return NULL; 603 } 604 605 if (initialize_structseq_dict( 606 desc, type->tp_dict, n_members, n_unnamed_members) < 0) { 607 Py_DECREF(type); 608 return NULL; 609 } 610 611 return type; 612} 613 614 615PyTypeObject * 616PyStructSequence_NewType(PyStructSequence_Desc *desc) 617{ 618 return _PyStructSequence_NewType(desc, 0); 619} 620