Mercurial > vim
changeset 33452:ed8db57d1034 v9.0.1980
patch 9.0.1980: win32: issues with stable python ABI
Commit: https://github.com/vim/vim/commit/119fdd9293f63614ed2ca60a78993466435db639
Author: Ken Takata <kentkt@csc.jp>
Date: Wed Oct 4 20:05:05 2023 +0200
patch 9.0.1980: win32: issues with stable python ABI
Problem: win32: issues with stable python ABI
Solution: if_python3,win32: Fix Python3 stable ABI
There were some issues in current stable ABI implementation on Windows:
* Python DLL name should be `python3.dll` instead of `python311.dll` and
so on. (See: https://docs.python.org/3/c-api/stable.html)
* Some non-stable API functions were used:
- `_PyObject_NextNotImplemented`
- `PyStdPrinter_Type`
* `reset_stdin()` and `hook_py_exit()` didn't work with `python3.dll`.
`python3.dll` is a special type of DLL called forwarder DLL.
It just forwards the functions to other DLL (e.g. `python311.dll`).
There were two issues regarding these functions:
- `python3.dll` doesn't have import tables. This caused a crash in
`get_imported_func_info()`. Add a check whether the specified DLL
has an import table.
- `reset_stdin()` and `hook_py_exit()` should be applied to the
forwarded DLL (e.g. `python311.dll`), not to `python3.dll`.
Check the export directory of `python3.dll` to find the forwarded
DLL and apply the functions to it.
closes: #13260
Signed-off-by: Christian Brabandt <cb@256bit.org>
Co-authored-by: Ken Takata <kentkt@csc.jp>
| author | Christian Brabandt <cb@256bit.org> |
|---|---|
| date | Wed, 04 Oct 2023 20:15:03 +0200 |
| parents | 843d48139b56 |
| children | 495fbedef23c |
| files | src/Make_cyg_ming.mak src/Make_mvc.mak src/if_python3.c src/os_win32.c src/proto/os_win32.pro src/version.c |
| diffstat | 6 files changed, 102 insertions(+), 15 deletions(-) [+] |
line wrap: on
line diff
--- a/src/Make_cyg_ming.mak +++ b/src/Make_cyg_ming.mak @@ -395,15 +395,20 @@ DYNAMIC_PYTHON3=yes ifndef PYTHON3_VER PYTHON3_VER=36 endif + ifeq ($(DYNAMIC_PYTHON3_STABLE_ABI),yes) +PYTHON3_NAME=python3 + else +PYTHON3_NAME=python$(PYTHON3_VER) + endif ifndef DYNAMIC_PYTHON3_DLL -DYNAMIC_PYTHON3_DLL=python$(PYTHON3_VER).dll +DYNAMIC_PYTHON3_DLL=$(PYTHON3_NAME).dll endif ifdef PYTHON3_HOME PYTHON3_HOME_DEF=-DPYTHON3_HOME=L\"$(PYTHON3_HOME)\" endif ifeq (no,$(DYNAMIC_PYTHON3)) -PYTHON3LIB=-L$(PYTHON3)/libs -lpython$(PYTHON3_VER) +PYTHON3LIB=-L$(PYTHON3)/libs -l$(PYTHON3_NAME) endif ifndef PYTHON3INC
--- a/src/Make_mvc.mak +++ b/src/Make_mvc.mak @@ -937,8 +937,13 @@ PYTHON_LIB = "$(PYTHON)\libs\python$(PYT ! ifndef PYTHON3_VER PYTHON3_VER = 36 ! endif +! if "$(DYNAMIC_PYTHON3_STABLE_ABI)" == "yes" +PYTHON3_NAME = python3 +! else +PYTHON3_NAME = python$(PYTHON3_VER) +! endif ! ifndef DYNAMIC_PYTHON3_DLL -DYNAMIC_PYTHON3_DLL = python$(PYTHON3_VER).dll +DYNAMIC_PYTHON3_DLL = $(PYTHON3_NAME).dll ! endif ! message Python3 requested (version $(PYTHON3_VER)) - root dir is "$(PYTHON3)" ! if "$(DYNAMIC_PYTHON3)" == "yes" @@ -953,13 +958,11 @@ CFLAGS = $(CFLAGS) -DDYNAMIC_PYTHON3 \ ! if "$(DYNAMIC_PYTHON3_STABLE_ABI)" == "yes" CFLAGS = $(CFLAGS) -DDYNAMIC_PYTHON3_STABLE_ABI PYTHON3_INC = $(PYTHON3_INC) -DPy_LIMITED_API=0x3080000 -PYTHON3_LIB = /nodefaultlib:python3.lib -! else -PYTHON3_LIB = /nodefaultlib:python$(PYTHON3_VER).lib ! endif +PYTHON3_LIB = /nodefaultlib:$(PYTHON3_NAME).lib ! else CFLAGS = $(CFLAGS) -DPYTHON3_DLL=\"$(DYNAMIC_PYTHON3_DLL)\" -PYTHON3_LIB = "$(PYTHON3)\libs\python$(PYTHON3_VER).lib" +PYTHON3_LIB = "$(PYTHON3)\libs\$(PYTHON3_NAME).lib" ! endif !endif
--- a/src/if_python3.c +++ b/src/if_python3.c @@ -242,7 +242,7 @@ static HINSTANCE hinstPy3 = 0; // Instan # if PY_VERSION_HEX >= 0x03040000 # define PyType_GetFlags py3_PyType_GetFlags # endif -#undef Py_BuildValue +# undef Py_BuildValue # define Py_BuildValue py3_Py_BuildValue # define Py_SetPythonHome py3_Py_SetPythonHome # define Py_Initialize py3_Py_Initialize @@ -251,7 +251,9 @@ static HINSTANCE hinstPy3 = 0; // Instan # define _Py_NoneStruct (*py3__Py_NoneStruct) # define _Py_FalseStruct (*py3__Py_FalseStruct) # define _Py_TrueStruct (*py3__Py_TrueStruct) -# define _PyObject_NextNotImplemented (*py3__PyObject_NextNotImplemented) +# ifndef USE_LIMITED_API +# define _PyObject_NextNotImplemented (*py3__PyObject_NextNotImplemented) +# endif # define PyModule_AddObject py3_PyModule_AddObject # define PyImport_AppendInittab py3_PyImport_AppendInittab # define PyImport_AddModule py3_PyImport_AddModule @@ -288,7 +290,9 @@ static HINSTANCE hinstPy3 = 0; // Instan # define PyFloat_AsDouble py3_PyFloat_AsDouble # define PyObject_GenericGetAttr py3_PyObject_GenericGetAttr # define PyType_Type (*py3_PyType_Type) -# define PyStdPrinter_Type (*py3_PyStdPrinter_Type) +# ifndef USE_LIMITED_API +# define PyStdPrinter_Type (*py3_PyStdPrinter_Type) +# endif # define PySlice_Type (*py3_PySlice_Type) # define PyFloat_Type (*py3_PyFloat_Type) # define PyNumber_Check (*py3_PyNumber_Check) @@ -449,7 +453,9 @@ static void (*py3_PyErr_Clear)(void); static PyObject* (*py3_PyErr_Format)(PyObject *, const char *, ...); static void (*py3_PyErr_PrintEx)(int); static PyObject*(*py3__PyObject_Init)(PyObject *, PyTypeObject *); +# ifndef USE_LIMITED_API static iternextfunc py3__PyObject_NextNotImplemented; +# endif static PyObject* py3__Py_NoneStruct; static PyObject* py3__Py_FalseStruct; static PyObject* py3__Py_TrueStruct; @@ -485,7 +491,9 @@ static PyObject* (*py3_PyObject_GenericG static PyObject* (*py3_PyType_GenericAlloc)(PyTypeObject *type, Py_ssize_t nitems); static PyObject* (*py3_PyType_GenericNew)(PyTypeObject *type, PyObject *args, PyObject *kwds); static PyTypeObject* py3_PyType_Type; +# ifndef USE_LIMITED_API static PyTypeObject* py3_PyStdPrinter_Type; +# endif static PyTypeObject* py3_PySlice_Type; static PyTypeObject* py3_PyFloat_Type; PyTypeObject* py3_PyBool_Type; @@ -633,7 +641,9 @@ static struct {"PyEval_SaveThread", (PYTHON_PROC*)&py3_PyEval_SaveThread}, {"_PyArg_Parse_SizeT", (PYTHON_PROC*)&py3_PyArg_Parse}, {"Py_IsInitialized", (PYTHON_PROC*)&py3_Py_IsInitialized}, +# ifndef USE_LIMITED_API {"_PyObject_NextNotImplemented", (PYTHON_PROC*)&py3__PyObject_NextNotImplemented}, +# endif {"_Py_NoneStruct", (PYTHON_PROC*)&py3__Py_NoneStruct}, {"_Py_FalseStruct", (PYTHON_PROC*)&py3__Py_FalseStruct}, {"_Py_TrueStruct", (PYTHON_PROC*)&py3__Py_TrueStruct}, @@ -681,7 +691,9 @@ static struct {"PyType_GenericAlloc", (PYTHON_PROC*)&py3_PyType_GenericAlloc}, {"PyType_GenericNew", (PYTHON_PROC*)&py3_PyType_GenericNew}, {"PyType_Type", (PYTHON_PROC*)&py3_PyType_Type}, +# ifndef USE_LIMITED_API {"PyStdPrinter_Type", (PYTHON_PROC*)&py3_PyStdPrinter_Type}, +# endif {"PySlice_Type", (PYTHON_PROC*)&py3_PySlice_Type}, {"PyFloat_Type", (PYTHON_PROC*)&py3_PyFloat_Type}, # if PY_VERSION_HEX < 0x030c00b0 @@ -1167,7 +1179,7 @@ reset_stdin(void) { FILE *(*py__acrt_iob_func)(unsigned) = NULL; FILE *(*pyfreopen)(const char *, const char *, FILE *) = NULL; - HINSTANCE hinst = hinstPy3; + HINSTANCE hinst = get_forwarded_dll(hinstPy3); if (hinst == NULL || is_stdin_readable()) return; @@ -1219,7 +1231,7 @@ hooked_exit(int ret) static void hook_py_exit(void) { - HINSTANCE hinst = hinstPy3; + HINSTANCE hinst = get_forwarded_dll(hinstPy3); if (hinst == NULL || orig_exit != NULL) return;
--- a/src/os_win32.c +++ b/src/os_win32.c @@ -626,15 +626,20 @@ get_imported_func_info(HINSTANCE hInst, PIMAGE_THUNK_DATA pIAT; // Import Address Table PIMAGE_THUNK_DATA pINT; // Import Name Table PIMAGE_IMPORT_BY_NAME pImpName; + DWORD ImpVA; if (pDOS->e_magic != IMAGE_DOS_SIGNATURE) return NULL; pPE = (PIMAGE_NT_HEADERS)(pImage + pDOS->e_lfanew); if (pPE->Signature != IMAGE_NT_SIGNATURE) return NULL; - pImpDesc = (PIMAGE_IMPORT_DESCRIPTOR)(pImage - + pPE->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT] - .VirtualAddress); + + ImpVA = pPE->OptionalHeader + .DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT].VirtualAddress; + if (ImpVA == 0) + return NULL; // No Import Table + pImpDesc = (PIMAGE_IMPORT_DESCRIPTOR)(pImage + ImpVA); + for (; pImpDesc->FirstThunk; ++pImpDesc) { if (!pImpDesc->OriginalFirstThunk) @@ -709,6 +714,65 @@ hook_dll_import_func(HINSTANCE hInst, co } #endif +#if defined(FEAT_PYTHON3) || defined(PROTO) +/* + * Check if the specified DLL is a function forwarder. + * If yes, return the instance of the forwarded DLL. + * If no, return the specified DLL. + * If error, return NULL. + * This assumes that the DLL forwards all the function to a single DLL. + */ + HINSTANCE +get_forwarded_dll(HINSTANCE hInst) +{ + PBYTE pImage = (PBYTE)hInst; + PIMAGE_DOS_HEADER pDOS = (PIMAGE_DOS_HEADER)hInst; + PIMAGE_NT_HEADERS pPE; + PIMAGE_EXPORT_DIRECTORY pExpDir; + DWORD ExpVA; + DWORD ExpSize; + LPDWORD pFunctionTable; + + if (pDOS->e_magic != IMAGE_DOS_SIGNATURE) + return NULL; + pPE = (PIMAGE_NT_HEADERS)(pImage + pDOS->e_lfanew); + if (pPE->Signature != IMAGE_NT_SIGNATURE) + return NULL; + + ExpVA = pPE->OptionalHeader + .DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT].VirtualAddress; + ExpSize = pPE->OptionalHeader + .DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT].Size; + if (ExpVA == 0) + return hInst; // No Export Directory + pExpDir = (PIMAGE_EXPORT_DIRECTORY)(pImage + ExpVA); + pFunctionTable = (LPDWORD)(pImage + pExpDir->AddressOfFunctions); + + if (pExpDir->NumberOfNames == 0) + return hInst; // No export names. + + // Check only the first entry. + if ((pFunctionTable[0] < ExpVA) || (pFunctionTable[0] >= ExpVA + ExpSize)) + // The first entry is not a function forwarder. + return hInst; + + // The first entry is a function forwarder. + // The name is represented as "DllName.FunctionName". + const char *name = (const char *)(pImage + pFunctionTable[0]); + const char *p = strchr(name, '.'); + if (p == NULL) + return hInst; + + // Extract DllName. + char buf[MAX_PATH]; + if (p - name + 1 > sizeof(buf)) + return NULL; + strncpy(buf, name, p - name); + buf[p - name] = '\0'; + return GetModuleHandleA(buf); +} +#endif + #if defined(DYNAMIC_GETTEXT) || defined(PROTO) # ifndef GETTEXT_DLL # define GETTEXT_DLL "libintl.dll"
--- a/src/proto/os_win32.pro +++ b/src/proto/os_win32.pro @@ -5,6 +5,7 @@ int mch_is_gui_executable(void); HINSTANCE find_imported_module_by_funcname(HINSTANCE hInst, const char *funcname); void *get_dll_import_func(HINSTANCE hInst, const char *funcname); void *hook_dll_import_func(HINSTANCE hInst, const char *funcname, const void *hook); +HINSTANCE get_forwarded_dll(HINSTANCE hInst); int dyn_libintl_init(void); void dyn_libintl_end(void); void PlatformId(void);