Add a "strict" option for map()

These two examples silently truncate the unmatched inputs:

>>> list(map(pow, [1, 2, 3], [2, 2, 2, 2]))
[1, 4, 9]
>>> list(map(pow, [1, 2, 3, 4, 5], [2, 2, 2, 2]))
[1, 4, 9, 16]

The current workaround is:

starmap(pow, zip(vec1, vec2, strict=True))

Ideally, map() should support this directly. The reasoning is the same reasoning that motivated the strict option for zip()

Linked PRs

PEP 618 says:

This PEP does not propose any changes to map, since the use of map with multiple iterable arguments is quite rare. However, this PEP’s ruling shall serve as precedent such a future discussion (should it occur).

If rejected, the feature is realistically not worth pursuing. If accepted, such a change to map should not require its own PEP (though, like all enhancements, its usefulness should be carefully considered). For consistency, it should follow same API and semantics debated here for zip.

I'm +1 on the change, personally. Unassigning myself though since I don't have bandwidth to work on it right now (it seems like a good issue for someone newer looking to contribute some C code).

I want to implement it, but I cant find the definition of map object. Can you tell me please where it is located?

See

cpython/Python/bltinmodule.c

Lines 1300 to 1527 in bf098d4

    
           /* map object ************************************************************/ 
        
           typedef struct { 
        
               PyObject_HEAD 
        
               PyObject *iters; 
        
               PyObject *func; 
        
           } mapobject; 
        
           static PyObject * 
        
           map_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 
        
           { 
        
               PyObject *it, *iters, *func; 
        
               mapobject *lz; 
        
               Py_ssize_t numargs, i; 
        
               if ((type == &PyMap_Type || type->tp_init == PyMap_Type.tp_init) && 
        
                   !_PyArg_NoKeywords("map", kwds)) 
        
                   return NULL; 
        
               numargs = PyTuple_Size(args); 
        
               if (numargs < 2) { 
        
                   PyErr_SetString(PyExc_TypeError, 
        
                      "map() must have at least two arguments."); 
        
                   return NULL; 
        
               } 
        
               iters = PyTuple_New(numargs-1); 
        
               if (iters == NULL) 
        
                   return NULL; 
        
               for (i=1 ; i<numargs ; i++) { 
        
                   /* Get iterator. */ 
        
                   it = PyObject_GetIter(PyTuple_GET_ITEM(args, i)); 
        
                   if (it == NULL) { 
        
                       Py_DECREF(iters); 
        
                       return NULL; 
        
                   } 
        
                   PyTuple_SET_ITEM(iters, i-1, it); 
        
               } 
        
               /* create mapobject structure */ 
        
               lz = (mapobject *)type->tp_alloc(type, 0); 
        
               if (lz == NULL) { 
        
                   Py_DECREF(iters); 
        
                   return NULL; 
        
               } 
        
               lz->iters = iters; 
        
               func = PyTuple_GET_ITEM(args, 0); 
        
               lz->func = Py_NewRef(func); 
        
               return (PyObject *)lz; 
        
           } 
        
           static PyObject * 
        
           map_vectorcall(PyObject *type, PyObject * const*args, 
        
                           size_t nargsf, PyObject *kwnames) 
        
           { 
        
               PyTypeObject *tp = _PyType_CAST(type); 
        
               if (tp == &PyMap_Type && !_PyArg_NoKwnames("map", kwnames)) { 
        
                   return NULL; 
        
               } 
        
               Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); 
        
               if (nargs < 2) { 
        
                   PyErr_SetString(PyExc_TypeError, 
        
                      "map() must have at least two arguments."); 
        
                   return NULL; 
        
               } 
        
               PyObject *iters = PyTuple_New(nargs-1); 
        
               if (iters == NULL) { 
        
                   return NULL; 
        
               } 
        
               for (int i=1; i<nargs; i++) { 
        
                   PyObject *it = PyObject_GetIter(args[i]); 
        
                   if (it == NULL) { 
        
                       Py_DECREF(iters); 
        
                       return NULL; 
        
                   } 
        
                   PyTuple_SET_ITEM(iters, i-1, it); 
        
               } 
        
               mapobject *lz = (mapobject *)tp->tp_alloc(tp, 0); 
        
               if (lz == NULL) { 
        
                   Py_DECREF(iters); 
        
                   return NULL; 
        
               } 
        
               lz->iters = iters; 
        
               lz->func = Py_NewRef(args[0]); 
        
               return (PyObject *)lz; 
        
           } 
        
           static void 
        
           map_dealloc(mapobject *lz) 
        
           { 
        
               PyObject_GC_UnTrack(lz); 
        
               Py_XDECREF(lz->iters); 
        
               Py_XDECREF(lz->func); 
        
               Py_TYPE(lz)->tp_free(lz); 
        
           } 
        
           static int 
        
           map_traverse(mapobject *lz, visitproc visit, void *arg) 
        
           { 
        
               Py_VISIT(lz->iters); 
        
               Py_VISIT(lz->func); 
        
               return 0; 
        
           } 
        
           static PyObject * 
        
           map_next(mapobject *lz) 
        
           { 
        
               PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; 
        
               PyObject **stack; 
        
               PyObject *result = NULL; 
        
               PyThreadState *tstate = _PyThreadState_GET(); 
        
               const Py_ssize_t niters = PyTuple_GET_SIZE(lz->iters); 
        
               if (niters <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { 
        
                   stack = small_stack; 
        
               } 
        
               else { 
        
                   stack = PyMem_Malloc(niters * sizeof(stack[0])); 
        
                   if (stack == NULL) { 
        
                       _PyErr_NoMemory(tstate); 
        
                       return NULL; 
        
                   } 
        
               } 
        
               Py_ssize_t nargs = 0; 
        
               for (Py_ssize_t i=0; i < niters; i++) { 
        
                   PyObject *it = PyTuple_GET_ITEM(lz->iters, i); 
        
                   PyObject *val = Py_TYPE(it)->tp_iternext(it); 
        
                   if (val == NULL) { 
        
                       goto exit; 
        
                   } 
        
                   stack[i] = val; 
        
                   nargs++; 
        
               } 
        
               result = _PyObject_VectorcallTstate(tstate, lz->func, stack, nargs, NULL); 
        
           exit: 
        
               for (Py_ssize_t i=0; i < nargs; i++) { 
        
                   Py_DECREF(stack[i]); 
        
               } 
        
               if (stack != small_stack) { 
        
                   PyMem_Free(stack); 
        
               } 
        
               return result; 
        
           } 
        
           static PyObject * 
        
           map_reduce(mapobject *lz, PyObject *Py_UNUSED(ignored)) 
        
           { 
        
               Py_ssize_t numargs = PyTuple_GET_SIZE(lz->iters); 
        
               PyObject *args = PyTuple_New(numargs+1); 
        
               Py_ssize_t i; 
        
               if (args == NULL) 
        
                   return NULL; 
        
               PyTuple_SET_ITEM(args, 0, Py_NewRef(lz->func)); 
        
               for (i = 0; i<numargs; i++){ 
        
                   PyObject *it = PyTuple_GET_ITEM(lz->iters, i); 
        
                   PyTuple_SET_ITEM(args, i+1, Py_NewRef(it)); 
        
               } 
        
               return Py_BuildValue("ON", Py_TYPE(lz), args); 
        
           } 
        
           static PyMethodDef map_methods[] = { 
        
               {"__reduce__", _PyCFunction_CAST(map_reduce), METH_NOARGS, reduce_doc}, 
        
               {NULL,           NULL}           /* sentinel */ 
        
           }; 
        
           PyDoc_STRVAR(map_doc, 
        
           "map(function, /, *iterables)\n\ 
        
           --\n\ 
        
           \n\ 
        
           Make an iterator that computes the function using arguments from\n\ 
        
           each of the iterables.  Stops when the shortest iterable is exhausted."); 
        
           PyTypeObject PyMap_Type = { 
        
               PyVarObject_HEAD_INIT(&PyType_Type, 0) 
        
               "map",                              /* tp_name */ 
        
               sizeof(mapobject),                  /* tp_basicsize */ 
        
               0,                                  /* tp_itemsize */ 
        
               /* methods */ 
        
               (destructor)map_dealloc,            /* tp_dealloc */ 
        
               0,                                  /* tp_vectorcall_offset */ 
        
               0,                                  /* tp_getattr */ 
        
               0,                                  /* tp_setattr */ 
        
               0,                                  /* tp_as_async */ 
        
               0,                                  /* tp_repr */ 
        
               0,                                  /* tp_as_number */ 
        
               0,                                  /* tp_as_sequence */ 
        
               0,                                  /* tp_as_mapping */ 
        
               0,                                  /* tp_hash */ 
        
               0,                                  /* tp_call */ 
        
               0,                                  /* tp_str */ 
        
               PyObject_GenericGetAttr,            /* tp_getattro */ 
        
               0,                                  /* tp_setattro */ 
        
               0,                                  /* tp_as_buffer */ 
        
               Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | 
        
                   Py_TPFLAGS_BASETYPE,            /* tp_flags */ 
        
               map_doc,                            /* tp_doc */ 
        
               (traverseproc)map_traverse,         /* tp_traverse */ 
        
               0,                                  /* tp_clear */ 
        
               0,                                  /* tp_richcompare */ 
        
               0,                                  /* tp_weaklistoffset */ 
        
               PyObject_SelfIter,                  /* tp_iter */ 
        
               (iternextfunc)map_next,     /* tp_iternext */ 
        
               map_methods,                        /* tp_methods */ 
        
               0,                                  /* tp_members */ 
        
               0,                                  /* tp_getset */ 
        
               0,                                  /* tp_base */ 
        
               0,                                  /* tp_dict */ 
        
               0,                                  /* tp_descr_get */ 
        
               0,                                  /* tp_descr_set */ 
        
               0,                                  /* tp_dictoffset */ 
        
               0,                                  /* tp_init */ 
        
               PyType_GenericAlloc,                /* tp_alloc */ 
        
               map_new,                            /* tp_new */ 
        
               PyObject_GC_Del,                    /* tp_free */ 
        
               .tp_vectorcall = (vectorcallfunc)map_vectorcall 
        
           };

The implementation should be almost identical to the implementation for zip, which is nice:

https://github.com/python/cpython/pull/20921/files#diff-e4fd8b8ee6a147f86c0719ff122aca6dfca36edbd4812c87892698b3b72e40a1

Brandt, should I credit you for the code I based my patch on?

Hey, I'm the guy who originally suggested strict for zip. I was just using map and wanting the same feature there, which prompted me to find this issue. I too am +1 on implementing it. If you do implement it, I'll be happy to write the documentation.

That would be much appreciated, although encukou doesn't think an example is necessary.

I see the documentation now, I agree an example isn't required. Good job.

This closes doors to another extension - adding keyword arguments to map.

Maybe it would be worth considering both at the same time and finding out what would be "joint optimum" to have performant & convenient solutions to both?:
a) map(func, arg1, arg2, c=arg3).
b) map(func, arg1, arg2, strict=True).

What about achieving map(func, arg1, arg2, c=arg3, strict=True)?

Of course, one can always convert func into positional only version for this, but I expect performance penalty for doing so is going to be much larger than performance benefit of map(..., strict) vs startmap(..., zip(..., strict))

Or is there a reason why adding keywords is out of the question?

gh-119793: Add optional length-checking to `map()` (GH-120471)

@nineteendo

Merged! Thank you @nineteendo for your work and patience, and thanks to all the reviewers who chimed in.

@nineteendo

Congratulations @nineteendo , @encukou and all reviewers for delivering this feature! I can't wait to use it.

pythongh-119793: Prefer `map(..., strict=True)` over starmap/zip in e…

The open doc followup issue looks like a no-brainer, but needs review and verification.

gh-119793: Prefer `map(..., strict=True)` over starmap/zip in examples (

The open doc followup issue looks like a no-brainer, but needs review and verification.

I merged the doc PR.

pythongh-119793: Add optional length-checking to `map()` (pythonGH-12…

pythongh-119793: Prefer `map(..., strict=True)` over starmap/zip in e…

pythongh-119793: Add optional length-checking to `map()` (pythonGH-12…

pythongh-119793: Prefer `map(..., strict=True)` over starmap/zip in e…

Navigation Menu

Repository navigation

Add a "strict" option for map() #119793

Linked PRs

3 remaining items

Metadata

Assignees

Labels

Projects

Milestone

Relationships

Development

	/* map object ************************************************************/

	typedef struct {
	PyObject_HEAD
	PyObject *iters;
	PyObject *func;
	} mapobject;

	static PyObject *
	map_new(PyTypeObject type, PyObject args, PyObject *kwds)
	{
	PyObject it, iters, *func;
	mapobject *lz;
	Py_ssize_t numargs, i;

	if ((type == &PyMap_Type \|\| type->tp_init == PyMap_Type.tp_init) &&
	!_PyArg_NoKeywords("map", kwds))
	return NULL;

	numargs = PyTuple_Size(args);
	if (numargs < 2) {
	PyErr_SetString(PyExc_TypeError,
	"map() must have at least two arguments.");
	return NULL;
	}

	iters = PyTuple_New(numargs-1);
	if (iters == NULL)
	return NULL;

	for (i=1 ; i<numargs ; i++) {
	/* Get iterator. */
	it = PyObject_GetIter(PyTuple_GET_ITEM(args, i));
	if (it == NULL) {
	Py_DECREF(iters);
	return NULL;
	}
	PyTuple_SET_ITEM(iters, i-1, it);
	}

	/* create mapobject structure */
	lz = (mapobject *)type->tp_alloc(type, 0);
	if (lz == NULL) {
	Py_DECREF(iters);
	return NULL;
	}
	lz->iters = iters;
	func = PyTuple_GET_ITEM(args, 0);
	lz->func = Py_NewRef(func);

	return (PyObject *)lz;
	}

	static PyObject *
	map_vectorcall(PyObject type, PyObject const*args,
	size_t nargsf, PyObject *kwnames)
	{
	PyTypeObject *tp = _PyType_CAST(type);
	if (tp == &PyMap_Type && !_PyArg_NoKwnames("map", kwnames)) {
	return NULL;
	}

	Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
	if (nargs < 2) {
	PyErr_SetString(PyExc_TypeError,
	"map() must have at least two arguments.");
	return NULL;
	}

	PyObject *iters = PyTuple_New(nargs-1);
	if (iters == NULL) {
	return NULL;
	}

	for (int i=1; i<nargs; i++) {
	PyObject *it = PyObject_GetIter(args[i]);
	if (it == NULL) {
	Py_DECREF(iters);
	return NULL;
	}
	PyTuple_SET_ITEM(iters, i-1, it);
	}

	mapobject lz = (mapobject )tp->tp_alloc(tp, 0);
	if (lz == NULL) {
	Py_DECREF(iters);
	return NULL;
	}
	lz->iters = iters;
	lz->func = Py_NewRef(args[0]);

	return (PyObject *)lz;
	}

	static void
	map_dealloc(mapobject *lz)
	{
	PyObject_GC_UnTrack(lz);
	Py_XDECREF(lz->iters);
	Py_XDECREF(lz->func);
	Py_TYPE(lz)->tp_free(lz);
	}

	static int
	map_traverse(mapobject lz, visitproc visit, void arg)
	{
	Py_VISIT(lz->iters);
	Py_VISIT(lz->func);
	return 0;
	}

	static PyObject *
	map_next(mapobject *lz)
	{
	PyObject *small_stack[_PY_FASTCALL_SMALL_STACK];
	PyObject **stack;
	PyObject *result = NULL;
	PyThreadState *tstate = _PyThreadState_GET();

	const Py_ssize_t niters = PyTuple_GET_SIZE(lz->iters);
	if (niters <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
	stack = small_stack;
	}
	else {
	stack = PyMem_Malloc(niters * sizeof(stack[0]));
	if (stack == NULL) {
	_PyErr_NoMemory(tstate);
	return NULL;
	}
	}

	Py_ssize_t nargs = 0;
	for (Py_ssize_t i=0; i < niters; i++) {
	PyObject *it = PyTuple_GET_ITEM(lz->iters, i);
	PyObject *val = Py_TYPE(it)->tp_iternext(it);
	if (val == NULL) {
	goto exit;
	}
	stack[i] = val;
	nargs++;
	}

	result = _PyObject_VectorcallTstate(tstate, lz->func, stack, nargs, NULL);

	exit:
	for (Py_ssize_t i=0; i < nargs; i++) {
	Py_DECREF(stack[i]);
	}
	if (stack != small_stack) {
	PyMem_Free(stack);
	}
	return result;
	}

	static PyObject *
	map_reduce(mapobject lz, PyObject Py_UNUSED(ignored))
	{
	Py_ssize_t numargs = PyTuple_GET_SIZE(lz->iters);
	PyObject *args = PyTuple_New(numargs+1);
	Py_ssize_t i;
	if (args == NULL)
	return NULL;
	PyTuple_SET_ITEM(args, 0, Py_NewRef(lz->func));
	for (i = 0; i<numargs; i++){
	PyObject *it = PyTuple_GET_ITEM(lz->iters, i);
	PyTuple_SET_ITEM(args, i+1, Py_NewRef(it));
	}

	return Py_BuildValue("ON", Py_TYPE(lz), args);
	}

	static PyMethodDef map_methods[] = {
	{"__reduce__", _PyCFunction_CAST(map_reduce), METH_NOARGS, reduce_doc},
	{NULL, NULL} /* sentinel */
	};


	PyDoc_STRVAR(map_doc,
	"map(function, /, *iterables)\n\
	--\n\
	\n\
	Make an iterator that computes the function using arguments from\n\
	each of the iterables. Stops when the shortest iterable is exhausted.");

	PyTypeObject PyMap_Type = {
	PyVarObject_HEAD_INIT(&PyType_Type, 0)
	"map", /* tp_name */
	sizeof(mapobject), /* tp_basicsize */
	0, /* tp_itemsize */
	/* methods */
	(destructor)map_dealloc, /* tp_dealloc */
	0, /* tp_vectorcall_offset */
	0, /* tp_getattr */
	0, /* tp_setattr */
	0, /* tp_as_async */
	0, /* tp_repr */
	0, /* tp_as_number */
	0, /* tp_as_sequence */
	0, /* tp_as_mapping */
	0, /* tp_hash */
	0, /* tp_call */
	0, /* tp_str */
	PyObject_GenericGetAttr, /* tp_getattro */
	0, /* tp_setattro */
	0, /* tp_as_buffer */
	Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_HAVE_GC \|
	Py_TPFLAGS_BASETYPE, /* tp_flags */
	map_doc, /* tp_doc */
	(traverseproc)map_traverse, /* tp_traverse */
	0, /* tp_clear */
	0, /* tp_richcompare */
	0, /* tp_weaklistoffset */
	PyObject_SelfIter, /* tp_iter */
	(iternextfunc)map_next, /* tp_iternext */
	map_methods, /* tp_methods */
	0, /* tp_members */
	0, /* tp_getset */
	0, /* tp_base */
	0, /* tp_dict */
	0, /* tp_descr_get */
	0, /* tp_descr_set */
	0, /* tp_dictoffset */
	0, /* tp_init */
	PyType_GenericAlloc, /* tp_alloc */
	map_new, /* tp_new */
	PyObject_GC_Del, /* tp_free */
	.tp_vectorcall = (vectorcallfunc)map_vectorcall
	};

Add a "strict" option for map() #119793

Description

Linked PRs

Activity

brandtbucher commented on May 30, 2024

SweetyAngel commented on May 30, 2024

zware commented on May 30, 2024

brandtbucher commented on May 30, 2024

nineteendo commented on Jun 13, 2024

cool-RR commented on Oct 3, 2024

nineteendo commented on Oct 3, 2024

cool-RR commented on Oct 3, 2024

dg-pb commented on Oct 3, 2024

3 remaining items

encukou commented on Nov 4, 2024

cool-RR commented on Nov 4, 2024

terryjreedy commented on Nov 4, 2024

vstinner commented on Nov 5, 2024

Metadata

Metadata

Assignees

Labels

Projects

Milestone

Relationships

Development

Issue actions