From f7ad7b2329f6c9120013cb8cad640f56dc5c3f44 Mon Sep 17 00:00:00 2001 From: Naveen M K Date: Tue, 20 Jun 2023 18:43:59 +0530 Subject: [PATCH 058/N] getpath: add support for mingw - always normalize the PREFIX to an absolute path - use `/` when MSYSTEM is defined --- Include/pylifecycle.h | 1 + Lib/test/test_getpath.py | 6 +++ Modules/getpath.c | 40 ++++++++++++++++++++ Modules/getpath.py | 81 ++++++++++++++++++++++++++-------------- Python/fileutils.c | 41 ++++++++++++-------- Python/pathconfig.c | 2 +- 6 files changed, 127 insertions(+), 44 deletions(-) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 9f842cb..6eb171d 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -21,6 +21,7 @@ PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); +PyAPI_FUNC(wchar_t) Py_GetAltSepW(const wchar_t *); PyAPI_FUNC(wchar_t) Py_GetSepW(const wchar_t *); PyAPI_FUNC(char) Py_GetSepA(const char *); diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 0c4ab74..f43877b 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -928,6 +928,9 @@ class MockNTNamespace(dict): except AttributeError: raise KeyError(key) from None + def normpath(self, path): + return ntpath.normpath(path) + def abspath(self, path): if self.isabs(path): return path @@ -1106,6 +1109,9 @@ class MockPosixNamespace(dict): except AttributeError: raise KeyError(key) from None + def normpath(self, path): + return path + def abspath(self, path): if self.isabs(path): return path diff --git a/Modules/getpath.c b/Modules/getpath.c index d0128b2..cf3fbdd 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -57,6 +57,25 @@ /* HELPER FUNCTIONS for getpath.py */ +static PyObject * +getpath_normpath(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } + Py_ssize_t len; + wchar_t *buffer = PyUnicode_AsWideCharString(pathobj, &len); + if (!buffer) { + return NULL; + } + r = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + PyMem_Free(buffer); + return r; +} + static PyObject * getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) { @@ -91,6 +110,12 @@ getpath_basename(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); +#ifdef ALTSEP + if (pos < 0) { + // try using altsep + pos = PyUnicode_FindChar(path, ALTSEP, 0, end, -1); + } +#endif if (pos < 0) { return Py_NewRef(path); } @@ -107,6 +132,12 @@ getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); +#ifdef ALTSEP + if (pos < 0) { + // try using altsep + pos = PyUnicode_FindChar(path, ALTSEP, 0, end, -1); + } +#endif if (pos < 0) { return PyUnicode_FromStringAndSize(NULL, 0); } @@ -561,6 +592,7 @@ done: static PyMethodDef getpath_methods[] = { + {"normpath", getpath_normpath, METH_VARARGS, NULL}, {"abspath", getpath_abspath, METH_VARARGS, NULL}, {"basename", getpath_basename, METH_VARARGS, NULL}, {"dirname", getpath_dirname, METH_VARARGS, NULL}, @@ -927,6 +959,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) #else !decode_to_dict(dict, "os_name", "posix") || #endif +#ifdef __MINGW32__ + !int_to_dict(dict, "is_mingw", 1) || +#else + !int_to_dict(dict, "is_mingw", 0) || +#endif #ifdef WITH_NEXT_FRAMEWORK !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 1) || #else @@ -958,6 +995,9 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) #endif #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || +#endif +#ifdef __MINGW32__ + !env_to_dict(dict, "ENV_MSYSTEM", 0) || #endif PyDict_SetItemString(dict, "__builtins__", PyEval_GetBuiltins()) < 0 ) { diff --git a/Modules/getpath.py b/Modules/getpath.py index 1f1bfcb..e743161 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -30,6 +30,7 @@ # ** Values known at compile time ** # os_name -- [in] one of 'nt', 'posix', 'darwin' +# is_mingw -- [in] True if targeting MinGW # PREFIX -- [in] sysconfig.get_config_var(...) # EXEC_PREFIX -- [in] sysconfig.get_config_var(...) # PYTHONPATH -- [in] sysconfig.get_config_var(...) @@ -52,6 +53,7 @@ # ENV_PYTHONHOME -- [in] getenv(...) # ENV_PYTHONEXECUTABLE -- [in] getenv(...) # ENV___PYVENV_LAUNCHER__ -- [in] getenv(...) +# ENV_MSYSTEM -- [in] getenv(...) # ** Values calculated at runtime ** # config -- [in/out] dict of the PyConfig structure @@ -187,8 +189,27 @@ if os_name == 'posix' or os_name == 'darwin': ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}{ABI_THREAD}.zip' DELIM = ':' SEP = '/' + ALTSEP = None -elif os_name == 'nt': +elif os_name == 'nt' and is_mingw: + BUILDDIR_TXT = 'pybuilddir.txt' + BUILD_LANDMARK = 'Modules/Setup.local' + DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] + VENV_LANDMARK = 'pyvenv.cfg' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + DELIM = ';' + if ENV_MSYSTEM: + SEP = '/' + ALTSEP = '\\' + else: + SEP = '\\' + ALTSEP = '/' + +elif os_name == 'nt': # MSVC BUILDDIR_TXT = 'pybuilddir.txt' BUILD_LANDMARK = f'{VPATH}\\Modules\\Setup.local' DEFAULT_PROGRAM_NAME = f'python' @@ -201,6 +222,7 @@ elif os_name == 'nt': WINREG_KEY = f'SOFTWARE\\Python\\PythonCore\\{PYWINVER}\\PythonPath' DELIM = ';' SEP = '\\' + ALTSEP = '/' # ****************************************************************************** @@ -213,6 +235,8 @@ def search_up(prefix, *landmarks, test=isfile): return prefix prefix = dirname(prefix) +def _normpath(p): + return normpath(p) if p is not None else None # ****************************************************************************** # READ VARIABLES FROM config @@ -264,10 +288,10 @@ if py_setpath: if not executable: executable = real_executable -if not executable and SEP in program_name: +if not executable and (SEP in program_name or + (ALTSEP and ALTSEP in program_name)): # Resolve partial path program_name against current directory executable = abspath(program_name) - if not executable: # All platforms default to real_executable if known at this # stage. POSIX does not set this value. @@ -502,15 +526,15 @@ if ((not home_was_set and real_executable_dir and not py_setpath) except (FileNotFoundError, PermissionError): if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)): build_prefix = joinpath(real_executable_dir, VPATH) - if os_name == 'nt': + if os_name == 'nt' and not is_mingw: # QUIRK: Windows builds need platstdlib_dir to be the executable # dir. Normally the builddir marker handles this, but in this # case we need to correct manually. platstdlib_dir = real_executable_dir if build_prefix: - if os_name == 'nt': - # QUIRK: No searching for more landmarks on Windows + if os_name == 'nt' and not is_mingw: + # QUIRK: No searching for more landmarks on MSVC build_stdlib_prefix = build_prefix else: build_stdlib_prefix = search_up(build_prefix, *BUILDSTDLIB_LANDMARKS) @@ -559,6 +583,9 @@ else: # First try to detect prefix by looking alongside our runtime library, if known if library and not prefix: library_dir = dirname(library) + if os_name == 'nt' and is_mingw: + # QUIRK: On Windows, mingw Python DLLs are in the bin directory + library_dir = joinpath(library_dir, '..') if ZIP_LANDMARK: if os_name == 'nt': # QUIRK: Windows does not search up for ZIP file @@ -605,7 +632,7 @@ else: # Detect exec_prefix by searching from executable for the platstdlib_dir if PLATSTDLIB_LANDMARK and not exec_prefix: - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows always assumed these were the same # gh-100320: Our PYDs are assumed to be relative to the Lib directory # (that is, prefix) rather than the executable (that is, executable_dir) @@ -615,7 +642,7 @@ else: if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: If DLLs is missing on Windows, don't warn, just assume # that they're in exec_prefix if not platstdlib_dir: @@ -653,7 +680,7 @@ else: if py_setpath: # If Py_SetPath was called then it overrides any existing search path - config['module_search_paths'] = py_setpath.split(DELIM) + config['module_search_paths'] = [_normpath(p) for p in py_setpath.split(DELIM)] config['module_search_paths_set'] = 1 elif not pythonpath_was_set: @@ -668,7 +695,7 @@ elif not pythonpath_was_set: pythonpath.append(abspath(p)) # Then add the default zip file - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows uses the library directory rather than the prefix if library: library_dir = dirname(library) @@ -681,7 +708,7 @@ elif not pythonpath_was_set: else: pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) - if os_name == 'nt' and use_environment and winreg: + if (not is_mingw) and os_name == 'nt' and use_environment and winreg: # QUIRK: Windows also lists paths in the registry. Paths are stored # as the default value of each subkey of # {HKCU,HKLM}\Software\Python\PythonCore\{winver}\PythonPath @@ -722,7 +749,7 @@ elif not pythonpath_was_set: if not platstdlib_dir and exec_prefix: platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK) - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows generates paths differently if platstdlib_dir: pythonpath.append(platstdlib_dir) @@ -740,7 +767,7 @@ elif not pythonpath_was_set: if platstdlib_dir: pythonpath.append(platstdlib_dir) - config['module_search_paths'] = pythonpath + config['module_search_paths'] = [_normpath(p) for p in pythonpath] config['module_search_paths_set'] = 1 @@ -750,8 +777,8 @@ elif not pythonpath_was_set: # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. -if os_name != 'nt' and build_prefix: - prefix = config.get('prefix') or PREFIX +if (os_name != 'nt' or is_mingw) and build_prefix: + prefix = config.get('prefix') or abspath(PREFIX) exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix @@ -775,23 +802,23 @@ if pth: warn("unsupported 'import' line in ._pth file") else: pythonpath.append(joinpath(pth_dir, line)) - config['module_search_paths'] = pythonpath + config['module_search_paths'] = [_normpath(p) for p in pythonpath] config['module_search_paths_set'] = 1 # ****************************************************************************** # UPDATE config FROM CALCULATED VALUES # ****************************************************************************** -config['program_name'] = program_name -config['home'] = home -config['executable'] = executable -config['base_executable'] = base_executable -config['prefix'] = prefix -config['exec_prefix'] = exec_prefix -config['base_prefix'] = base_prefix or prefix -config['base_exec_prefix'] = base_exec_prefix or exec_prefix +config['program_name'] = _normpath(program_name) +config['home'] = _normpath(home) +config['executable'] = _normpath(executable) +config['base_executable'] = _normpath(base_executable) +config['prefix'] = _normpath(prefix) +config['exec_prefix'] = _normpath(exec_prefix) +config['base_prefix'] = _normpath(base_prefix or prefix) +config['base_exec_prefix'] = _normpath(base_exec_prefix or exec_prefix) -config['platlibdir'] = platlibdir +config['platlibdir'] = _normpath(platlibdir) # test_embed expects empty strings, not None -config['stdlib_dir'] = stdlib_dir or '' -config['platstdlib_dir'] = platstdlib_dir or '' +config['stdlib_dir'] = _normpath(stdlib_dir or '') +config['platstdlib_dir'] = _normpath(platstdlib_dir or '') diff --git a/Python/fileutils.c b/Python/fileutils.c index 0ae2a2e..a6e43e7 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2186,7 +2186,11 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) } #ifdef MS_WINDOWS - return _PyOS_getfullpathname(path, abspath_p); + if (_PyOS_getfullpathname(path, abspath_p) < 0){ + return -1; + } + *abspath_p = _Py_normpath(*abspath_p, -1); + return 0; #else wchar_t cwd[MAXPATHLEN + 1]; cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; @@ -2491,11 +2495,16 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) wchar_t *minP2 = path; // the beginning of the destination range wchar_t lastC = L'\0'; // the last ljusted character, p2[-1] in most cases + const wchar_t sep = Py_GetSepW(NULL); +#ifdef ALTSEP + const wchar_t altsep = Py_GetAltSepW(NULL); +#endif + #define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) #ifdef ALTSEP -#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP) +#define IS_SEP(x) (*(x) == sep || *(x) == altsep) #else -#define IS_SEP(x) (*(x) == SEP) +#define IS_SEP(x) (*(x) == sep) #endif #define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) @@ -2508,15 +2517,15 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) p2 = p1; #else for (; p2 < p1; ++p2) { - if (*p2 == ALTSEP) { - *p2 = SEP; + if (*p2 == altsep) { + *p2 = sep; } } #endif minP2 = p2 - 1; lastC = *minP2; #ifdef MS_WINDOWS - if (lastC != SEP) { + if (lastC != sep) { minP2++; } #endif @@ -2525,8 +2534,8 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) // Skip leading '.\' lastC = *++p1; #ifdef ALTSEP - if (lastC == ALTSEP) { - lastC = SEP; + if (lastC == altsep) { + lastC = sep; } #endif while (IS_SEP(p1)) { @@ -2538,18 +2547,18 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) for (; !IS_END(p1); ++p1) { wchar_t c = *p1; #ifdef ALTSEP - if (c == ALTSEP) { - c = SEP; + if (c == altsep) { + c = sep; } #endif - if (lastC == SEP) { + if (lastC == sep) { if (c == L'.') { int sep_at_1 = SEP_OR_END(&p1[1]); int sep_at_2 = !sep_at_1 && SEP_OR_END(&p1[2]); if (sep_at_2 && p1[1] == L'.') { wchar_t *p3 = p2; - while (p3 != minP2 && *--p3 == SEP) { } - while (p3 != minP2 && *(p3 - 1) != SEP) { --p3; } + while (p3 != minP2 && *--p3 == sep) { } + while (p3 != minP2 && *(p3 - 1) != sep) { --p3; } if (p2 == minP2 || (p3[0] == L'.' && p3[1] == L'.' && IS_SEP(&p3[2]))) { @@ -2558,7 +2567,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) *p2++ = L'.'; *p2++ = L'.'; lastC = L'.'; - } else if (p3[0] == SEP) { + } else if (p3[0] == sep) { // Absolute path, so absorb segment p2 = p3 + 1; } else { @@ -2569,7 +2578,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) } else { *p2++ = lastC = c; } - } else if (c == SEP) { + } else if (c == sep) { } else { *p2++ = lastC = c; } @@ -2579,7 +2588,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) } *p2 = L'\0'; if (p2 != minP2) { - while (--p2 != minP2 && *p2 == SEP) { + while (--p2 != minP2 && *p2 == sep) { *p2 = L'\0'; } } else { diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 0033e18..8653915 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -127,7 +127,7 @@ Py_GetSepW(const wchar_t *name) return sep; } -static wchar_t +wchar_t Py_GetAltSepW(const wchar_t *name) { char sep = Py_GetSepW(name);