# HG changeset patch # User Bram Moolenaar # Date 1371039636 -7200 # Node ID 96e154e825a7441274a615e7d7fdbe68e5534ee9 # Parent 6c1f3a6714bd751b7e00139ebc6d1c3cc7a6ec70 updated for version 7.3.1172 Problem: Python 2: loading modules doesn't work well. Solution: Fix the code. Add more tests. (ZyX) diff --git a/Filelist b/Filelist --- a/Filelist +++ b/Filelist @@ -87,6 +87,8 @@ SRC_ALL = \ src/testdir/python2/*.py \ src/testdir/python3/*.py \ src/testdir/pythonx/*.py \ + src/testdir/python_after/*.py \ + src/testdir/python_before/*.py \ src/proto.h \ src/proto/blowfish.pro \ src/proto/buffer.pro \ diff --git a/runtime/doc/if_pyth.txt b/runtime/doc/if_pyth.txt --- a/runtime/doc/if_pyth.txt +++ b/runtime/doc/if_pyth.txt @@ -315,52 +315,53 @@ vim.path_hooks in sys.path_hooks python {rtp}/python2 (or python3) and {rtp}/pythonx (for both python versions) for each {rtp} found in 'runtimepath'. -Implementation for python 2 is the following: usual importing code with empty -lists in place of sys.path_hooks and sys.meta_path. Code is similar to the -below, but written in C: > +Implementation for python 2 is similar to the following, but written in C: > - # Assuming vim variable is already accessible and is set to the current - # module + from imp import find_module, load_module + import vim import sys - def find_module(fullname): - return vim + class VimModuleLoader(object): + def __init__(self, module): + self.module = module - def load_module(fullname): - # see vim._get_paths below - new_path = _get_paths() + def load_module(self, fullname, path=None): + return self.module - try: old_path = sys.path - except: pass - try: old_meta_path = sys.meta_path - except: pass - try: old_path_hooks = sys.path_hooks - except: pass - - sys.meta_path = [] - sys.path_hooks = sys.meta_path - sys.path = new_path + def _find_module(fullname, oldtail, path): + idx = oldtail.find('.') + if idx > 0: + name = oldtail[:idx] + tail = oldtail[idx+1:] + fmr = find_module(name, path) + module = load_module(fullname[:-len(oldtail)] + name, *fmr) + return _find_module(fullname, tail, module.__path__) + else: + fmr = find_module(fullname, path) + return load_module(fullname, *fmr) - try: - exec ('import ' + fullname + ' as m') # No actual exec in C code - return m - finally: - e = None - try: sys.path = old_path - except Exception as e: pass - try: sys.meta_path = old_meta_path - except Exception as e: pass - try: sys.path_hooks = old_path_hooks - except Exception as e: pass - if e: - raise e + # It uses vim module itself in place of VimPathFinder class: it does not + # matter for python which object has find_module function attached to as + # an attribute. + class VimPathFinder(object): + def find_module(cls, fullname, path=None): + try: + return VimModuleLoader(_find_module(fullname, fullname, path or vim._get_paths())) + except ImportError: + return None + find_module = classmethod(find_module) - def path_hook(d): - if d == VIM_SPECIAL_PATH: - return vim - raise ImportError + def load_module(cls, fullname, path=None): + return _find_module(fullname, fullname, path or vim._get_paths()) + load_module = classmethod(load_module) - sys.path_hooks.append(path_hook) + def hook(path): + if path == vim.VIM_SPECIAL_PATH: + return VimPathFinder + else: + raise ImportError + + sys.path_hooks.append(hook) Implementation for python 3 is cleaner: code is similar to the following, but, again, written in C: > @@ -395,14 +396,13 @@ vim.VIM_SPECIAL_PATH *python-VIM_SPE Note: you must not use value of this constant directly, always use vim.VIM_SPECIAL_PATH object. -vim.load_module(name) *python-load_module* vim.find_module(...) *python-find_module* vim.path_hook(path) *python-path_hook* Methods or objects used to implement path loading as described above. You should not be using any of these directly except for vim.path_hook in case you need to do something with sys.meta_path. It is not guaranteed that any of the objects will exist in the future vim - versions. In fact, load_module and find_module methods do not exists + versions. In fact, find_module methods do not exists in python3. vim._get_paths *python-_get_paths* diff --git a/src/if_py_both.h b/src/if_py_both.h --- a/src/if_py_both.h +++ b/src/if_py_both.h @@ -940,7 +940,6 @@ static struct PyMethodDef VimMethods[] = {"foreach_rtp", VimForeachRTP, METH_VARARGS, "Call given callable for each path in &rtp"}, #if PY_MAJOR_VERSION < 3 {"find_module", FinderFindModule, METH_VARARGS, "Internal use only, returns loader object for any input it receives"}, - {"load_module", LoaderLoadModule, METH_VARARGS, "Internal use only, tries importing the given module from &rtp by temporary mocking sys.path (to an rtp-based one) and unsetting sys.meta_path and sys.path_hooks"}, #endif {"path_hook", VimPathHook, METH_VARARGS, "Hook function to install in sys.path_hooks"}, {"_get_paths", (PyCFunction)Vim_GetPaths, METH_NOARGS, "Get &rtp-based additions to sys.path"}, @@ -5195,6 +5194,13 @@ typedef struct PyObject_HEAD } FinderObject; static PyTypeObject FinderType; +#else +typedef struct +{ + PyObject_HEAD + PyObject *module; +} LoaderObject; +static PyTypeObject LoaderType; #endif static void @@ -5444,6 +5450,8 @@ init_types() PYTYPE_READY(OutputType); #if PY_MAJOR_VERSION >= 3 PYTYPE_READY(FinderType); +#else + PYTYPE_READY(LoaderType); #endif return 0; } @@ -5570,6 +5578,8 @@ static struct object_constant { {"Options", (PyObject *)&OptionsType}, #if PY_MAJOR_VERSION >= 3 {"Finder", (PyObject *)&FinderType}, +#else + {"Loader", (PyObject *)&LoaderType}, #endif }; @@ -5666,6 +5676,9 @@ populate_module(PyObject *m, object_adde ADD_CHECKED_OBJECT(m, "_find_module", (py_find_module = PyObject_GetAttrString(path_finder, "find_module"))); +#else + ADD_OBJECT(m, "_find_module", py_find_module); + ADD_OBJECT(m, "_load_module", py_load_module); #endif return 0; diff --git a/src/if_python.c b/src/if_python.c --- a/src/if_python.c +++ b/src/if_python.c @@ -150,6 +150,7 @@ struct PyMethodDef { Py_ssize_t a; }; # undef Py_InitModule4 # undef Py_InitModule4_64 # undef PyObject_CallMethod +# undef PyObject_CallFunction /* * Wrapper defines @@ -219,6 +220,7 @@ struct PyMethodDef { Py_ssize_t a; }; # define PyObject_HasAttrString dll_PyObject_HasAttrString # define PyObject_SetAttrString dll_PyObject_SetAttrString # define PyObject_CallFunctionObjArgs dll_PyObject_CallFunctionObjArgs +# define PyObject_CallFunction dll_PyObject_CallFunction # define PyObject_Call dll_PyObject_Call # define PyString_AsString dll_PyString_AsString # define PyString_AsStringAndSize dll_PyString_AsStringAndSize @@ -357,6 +359,7 @@ static PyObject* (*dll_PyObject_GetAttrS static int (*dll_PyObject_HasAttrString)(PyObject *, const char *); static PyObject* (*dll_PyObject_SetAttrString)(PyObject *, const char *, PyObject *); static PyObject* (*dll_PyObject_CallFunctionObjArgs)(PyObject *, ...); +static PyObject* (*dll_PyObject_CallFunction)(PyObject *, char *, ...); static PyObject* (*dll_PyObject_Call)(PyObject *, PyObject *, PyObject *); static char*(*dll_PyString_AsString)(PyObject *); static int(*dll_PyString_AsStringAndSize)(PyObject *, char **, int *); @@ -528,6 +531,7 @@ static struct {"PyObject_HasAttrString", (PYTHON_PROC*)&dll_PyObject_HasAttrString}, {"PyObject_SetAttrString", (PYTHON_PROC*)&dll_PyObject_SetAttrString}, {"PyObject_CallFunctionObjArgs", (PYTHON_PROC*)&dll_PyObject_CallFunctionObjArgs}, + {"PyObject_CallFunction", (PYTHON_PROC*)&dll_PyObject_CallFunction}, {"PyObject_Call", (PYTHON_PROC*)&dll_PyObject_Call}, {"PyString_AsString", (PYTHON_PROC*)&dll_PyString_AsString}, {"PyString_AsStringAndSize", (PYTHON_PROC*)&dll_PyString_AsStringAndSize}, @@ -748,10 +752,12 @@ static PyObject *DictionaryGetattr(PyObj static PyObject *ListGetattr(PyObject *, char *); static PyObject *FunctionGetattr(PyObject *, char *); -static PyObject *LoaderLoadModule(PyObject *, PyObject *); static PyObject *FinderFindModule(PyObject *, PyObject *); static PyObject *VimPathHook(PyObject *, PyObject *); +static PyObject *py_find_module; +static PyObject *py_load_module; + #ifndef Py_VISIT # define Py_VISIT(obj) visit(obj, arg) #endif @@ -1376,16 +1382,127 @@ python_tabpage_free(tabpage_T *tab) } #endif + static void +LoaderDestructor(LoaderObject *self) +{ + Py_DECREF(self->module); + DESTRUCTOR_FINISH(self); +} + static PyObject * -LoaderLoadModule(PyObject *self, PyObject *args) +LoaderLoadModule(LoaderObject *self, PyObject *args UNUSED) +{ + PyObject *r = self->module; + + Py_INCREF(r); + return r; +} + +static struct PyMethodDef LoaderMethods[] = { + /* name, function, calling, doc */ + {"load_module", (PyCFunction)LoaderLoadModule, METH_VARARGS, ""}, + { NULL, NULL, 0, NULL} +}; + + static PyObject * +call_load_module(char *name, int len, PyObject *find_module_result) +{ + PyObject *fd, *pathname, *description; + + if (!PyTuple_Check(find_module_result) + || PyTuple_GET_SIZE(find_module_result) != 3) + { + PyErr_SetString(PyExc_TypeError, + _("expected 3-tuple as imp.find_module() result")); + return NULL; + } + + if (!(fd = PyTuple_GET_ITEM(find_module_result, 0)) + || !(pathname = PyTuple_GET_ITEM(find_module_result, 1)) + || !(description = PyTuple_GET_ITEM(find_module_result, 2))) + { + PyErr_SetString(PyExc_RuntimeError, + _("internal error: imp.find_module returned tuple with NULL")); + return NULL; + } + + return PyObject_CallFunction(py_load_module, + "s#OOO", name, len, fd, pathname, description); +} + + static PyObject * +find_module(char *fullname, char *tail, PyObject *new_path) +{ + PyObject *find_module_result; + PyObject *module; + char *dot; + + if ((dot = (char *) vim_strchr((char_u *) tail, '.'))) + { + /* + * There is a dot in the name: call find_module recursively without the + * first component + */ + PyObject *newest_path; + int partlen = (int) (dot - 1 - tail); + + if (!(find_module_result = PyObject_CallFunction(py_find_module, + "s#O", tail, partlen, new_path))) + return NULL; + + if (!(module = call_load_module( + fullname, + ((int) (tail - fullname)) + partlen, + find_module_result))) + { + Py_DECREF(find_module_result); + return NULL; + } + + Py_DECREF(find_module_result); + + if (!(newest_path = PyObject_GetAttrString(module, "__path__"))) + { + Py_DECREF(module); + return NULL; + } + + Py_DECREF(module); + + module = find_module(fullname, dot + 1, newest_path); + + Py_DECREF(newest_path); + + return module; + } + else + { + if (!(find_module_result = PyObject_CallFunction(py_find_module, + "sO", tail, new_path))) + return NULL; + + if (!(module = call_load_module( + fullname, + STRLEN(fullname), + find_module_result))) + { + Py_DECREF(find_module_result); + return NULL; + } + + Py_DECREF(find_module_result); + + return module; + } +} + + static PyObject * +FinderFindModule(PyObject *self, PyObject *args) { char *fullname; - PyObject *path; - PyObject *meta_path; - PyObject *path_hooks; + PyObject *module; PyObject *new_path; - PyObject *r; - PyObject *new_list; + LoaderObject *loader; if (!PyArg_ParseTuple(args, "s", &fullname)) return NULL; @@ -1393,73 +1510,25 @@ LoaderLoadModule(PyObject *self, PyObjec if (!(new_path = Vim_GetPaths(self))) return NULL; - if (!(new_list = PyList_New(0))) - return NULL; + module = find_module(fullname, fullname, new_path); -#define GET_SYS_OBJECT(objstr, obj) \ - obj = PySys_GetObject(objstr); \ - PyErr_Clear(); \ - Py_XINCREF(obj); + Py_DECREF(new_path); - GET_SYS_OBJECT("meta_path", meta_path); - if (PySys_SetObject("meta_path", new_list)) + if (!module) { - Py_XDECREF(meta_path); - Py_DECREF(new_list); - return NULL; - } - Py_DECREF(new_list); /* Now it becomes a reference borrowed from - sys.meta_path */ - -#define RESTORE_SYS_OBJECT(objstr, obj) \ - if (obj) \ - { \ - PySys_SetObject(objstr, obj); \ - Py_DECREF(obj); \ + Py_INCREF(Py_None); + return Py_None; } - GET_SYS_OBJECT("path_hooks", path_hooks); - if (PySys_SetObject("path_hooks", new_list)) + if (!(loader = PyObject_NEW(LoaderObject, &LoaderType))) { - RESTORE_SYS_OBJECT("meta_path", meta_path); - Py_XDECREF(path_hooks); + Py_DECREF(module); return NULL; } - GET_SYS_OBJECT("path", path); - if (PySys_SetObject("path", new_path)) - { - RESTORE_SYS_OBJECT("meta_path", meta_path); - RESTORE_SYS_OBJECT("path_hooks", path_hooks); - Py_XDECREF(path); - return NULL; - } - Py_DECREF(new_path); - - r = PyImport_ImportModule(fullname); - - RESTORE_SYS_OBJECT("meta_path", meta_path); - RESTORE_SYS_OBJECT("path_hooks", path_hooks); - RESTORE_SYS_OBJECT("path", path); + loader->module = module; - if (PyErr_Occurred()) - { - Py_XDECREF(r); - return NULL; - } - - return r; -} - - static PyObject * -FinderFindModule(PyObject *self UNUSED, PyObject *args UNUSED) -{ - /* - * Don't bother actually finding the module, it is delegated to the "loader" - * object (which is basically the same object: vim module). - */ - Py_INCREF(vim_module); - return vim_module; + return (PyObject *) loader; } static PyObject * @@ -1483,7 +1552,34 @@ VimPathHook(PyObject *self UNUSED, PyObj PythonMod_Init(void) { /* The special value is removed from sys.path in Python_Init(). */ - static char *(argv[2]) = {"/must>not&exist/foo", NULL}; + static char *(argv[2]) = {"/must>not&exist/foo", NULL}; + PyObject *imp; + + if (!(imp = PyImport_ImportModule("imp"))) + return -1; + + if (!(py_find_module = PyObject_GetAttrString(imp, "find_module"))) + { + Py_DECREF(imp); + return -1; + } + + if (!(py_load_module = PyObject_GetAttrString(imp, "load_module"))) + { + Py_DECREF(py_find_module); + Py_DECREF(imp); + return -1; + } + + Py_DECREF(imp); + + vim_memset(&LoaderType, 0, sizeof(LoaderType)); + LoaderType.tp_name = "vim.Loader"; + LoaderType.tp_basicsize = sizeof(LoaderObject); + LoaderType.tp_flags = Py_TPFLAGS_DEFAULT; + LoaderType.tp_doc = "vim message object"; + LoaderType.tp_methods = LoaderMethods; + LoaderType.tp_dealloc = (destructor)LoaderDestructor; if (init_types()) return -1; diff --git a/src/testdir/python2/module.py b/src/testdir/python2/module.py --- a/src/testdir/python2/module.py +++ b/src/testdir/python2/module.py @@ -1,1 +1,2 @@ +import before_1 dir = '2' diff --git a/src/testdir/python3/module.py b/src/testdir/python3/module.py --- a/src/testdir/python3/module.py +++ b/src/testdir/python3/module.py @@ -1,1 +1,2 @@ +import before_1 dir = '3' diff --git a/src/testdir/python_after/after.py b/src/testdir/python_after/after.py new file mode 100644 --- /dev/null +++ b/src/testdir/python_after/after.py @@ -0,0 +1,2 @@ +import before_2 +dir = "after" diff --git a/src/testdir/python_before/before.py b/src/testdir/python_before/before.py new file mode 100644 --- /dev/null +++ b/src/testdir/python_before/before.py @@ -0,0 +1,1 @@ +dir = "before" diff --git a/src/testdir/test86.in b/src/testdir/test86.in --- a/src/testdir/test86.in +++ b/src/testdir/test86.in @@ -8,6 +8,7 @@ See http://svn.python.org/view/python/tr STARTTEST :so small.vim :set encoding=latin1 +:set noswapfile :if !has('python') | e! test.ok | wq! test.out | endif :lang C :py import vim @@ -1071,10 +1072,16 @@ EOF :" :" Test import py << EOF +sys.path.insert(0, os.path.join(os.getcwd(), 'python_before')) +sys.path.append(os.path.join(os.getcwd(), 'python_after')) vim.options['rtp'] = os.getcwd().replace(',', '\\,').replace('\\', '\\\\') from module import dir as d from modulex import ddir cb.append(d + ',' + ddir) +import before +cb.append(before.dir) +import after +cb.append(after.dir) EOF :" :" Test exceptions diff --git a/src/testdir/test86.ok b/src/testdir/test86.ok --- a/src/testdir/test86.ok +++ b/src/testdir/test86.ok @@ -1084,6 +1084,8 @@ vim.current.window = True:(, TypeError('expected vim.TabPage object',)) vim.current.xxx = True:(, AttributeError('xxx',)) 2,xx +before +after vim.command("throw 'abc'"):(, error('abc',)) Exe("throw 'def'"):(, error('def',)) vim.eval("Exe('throw ''ghi''')"):(, error('ghi',)) diff --git a/src/testdir/test87.in b/src/testdir/test87.in --- a/src/testdir/test87.in +++ b/src/testdir/test87.in @@ -2,6 +2,7 @@ Tests for various python features. v STARTTEST :so small.vim +:set noswapfile :if !has('python3') | e! test.ok | wq! test.out | endif :lang C :py3 import vim @@ -1038,10 +1039,16 @@ EOF :" :" Test import py3 << EOF +sys.path.insert(0, os.path.join(os.getcwd(), 'python_before')) +sys.path.append(os.path.join(os.getcwd(), 'python_after')) vim.options['rtp'] = os.getcwd().replace(',', '\\,').replace('\\', '\\\\') from module import dir as d from modulex import ddir cb.append(d + ',' + ddir) +import before +cb.append(before.dir) +import after +cb.append(after.dir) EOF :" :" Test exceptions diff --git a/src/testdir/test87.ok b/src/testdir/test87.ok --- a/src/testdir/test87.ok +++ b/src/testdir/test87.ok @@ -1093,6 +1093,8 @@ vim.current.window = True:(, TypeError('expected vim.TabPage object',)) vim.current.xxx = True:(, AttributeError('xxx',)) 3,xx +before +after vim.command("throw 'abc'"):(, error('abc',)) Exe("throw 'def'"):(, error('def',)) vim.eval("Exe('throw ''ghi''')"):(, error('ghi',)) diff --git a/src/version.c b/src/version.c --- a/src/version.c +++ b/src/version.c @@ -729,6 +729,8 @@ static char *(features[]) = static int included_patches[] = { /* Add new patch number below this line */ /**/ + 1172, +/**/ 1171, /**/ 1170,