juc500 정상동작

This commit is contained in:
Macbook
2026-07-21 21:04:25 +09:00
parent ff60cee1cc
commit 1388668b56
2917 changed files with 708021 additions and 651 deletions
@@ -0,0 +1 @@
import __editable___juc500_xfer_0_1_0_finder; __editable___juc500_xfer_0_1_0_finder.install()
@@ -0,0 +1,85 @@
from __future__ import annotations
import sys
from importlib.machinery import ModuleSpec, PathFinder
from importlib.machinery import all_suffixes as module_suffixes
from importlib.util import spec_from_file_location
from itertools import chain
from pathlib import Path
MAPPING: dict[str, str] = {'juc500_xfer': 'C:\\dev\\juc500\\juc500\\juc500_xfer'}
NAMESPACES: dict[str, list[str]] = {'juc500_xfer.static': ['C:\\dev\\juc500\\juc500\\juc500_xfer\\static']}
PATH_PLACEHOLDER = '__editable__.juc500_xfer-0.1.0.finder' + ".__path_hook__"
class _EditableFinder: # MetaPathFinder
@classmethod
def find_spec(cls, fullname: str, path=None, target=None) -> ModuleSpec | None: # type: ignore
# Top-level packages and modules (we know these exist in the FS)
if fullname in MAPPING:
pkg_path = MAPPING[fullname]
return cls._find_spec(fullname, Path(pkg_path))
# Handle immediate children modules (required for namespaces to work)
# To avoid problems with case sensitivity in the file system we delegate
# to the importlib.machinery implementation.
parent, _, child = fullname.rpartition(".")
if parent and parent in MAPPING:
return PathFinder.find_spec(fullname, path=[MAPPING[parent]])
# Other levels of nesting should be handled automatically by importlib
# using the parent path.
return None
@classmethod
def _find_spec(cls, fullname: str, candidate_path: Path) -> ModuleSpec | None:
init = candidate_path / "__init__.py"
candidates = (candidate_path.with_suffix(x) for x in module_suffixes())
for candidate in chain([init], candidates):
if candidate.exists():
return spec_from_file_location(fullname, candidate)
return None
class _EditableNamespaceFinder: # PathEntryFinder
@classmethod
def _path_hook(cls, path) -> type[_EditableNamespaceFinder]:
if path == PATH_PLACEHOLDER:
return cls
raise ImportError
@classmethod
def _paths(cls, fullname: str) -> list[str]:
paths = NAMESPACES[fullname]
if not paths and fullname in MAPPING:
paths = [MAPPING[fullname]]
# Always add placeholder, for 2 reasons:
# 1. __path__ cannot be empty for the spec to be considered namespace.
# 2. In the case of nested namespaces, we need to force
# import machinery to query _EditableNamespaceFinder again.
return [*paths, PATH_PLACEHOLDER]
@classmethod
def find_spec(cls, fullname: str, target=None) -> ModuleSpec | None: # type: ignore
if fullname in NAMESPACES:
spec = ModuleSpec(fullname, None, is_package=True)
spec.submodule_search_locations = cls._paths(fullname)
return spec
return None
@classmethod
def find_module(cls, _fullname) -> None:
return None
def install():
if not any(finder == _EditableFinder for finder in sys.meta_path):
sys.meta_path.append(_EditableFinder)
if not NAMESPACES:
return
if not any(hook == _EditableNamespaceFinder._path_hook for hook in sys.path_hooks):
# PathEntryFinder is needed to create NamespaceSpec without private APIS
sys.path_hooks.append(_EditableNamespaceFinder._path_hook)
if PATH_PLACEHOLDER not in sys.path:
sys.path.append(PATH_PLACEHOLDER) # Used just to trigger the path hook
@@ -0,0 +1,239 @@
# don't import any costly modules
import os
import sys
report_url = (
"https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml"
)
def warn_distutils_present():
if 'distutils' not in sys.modules:
return
import warnings
warnings.warn(
"Distutils was imported before Setuptools, but importing Setuptools "
"also replaces the `distutils` module in `sys.modules`. This may lead "
"to undesirable behaviors or errors. To avoid these issues, avoid "
"using distutils directly, ensure that setuptools is installed in the "
"traditional way (e.g. not an editable install), and/or make sure "
"that setuptools is always imported before distutils."
)
def clear_distutils():
if 'distutils' not in sys.modules:
return
import warnings
warnings.warn(
"Setuptools is replacing distutils. Support for replacing "
"an already imported distutils is deprecated. In the future, "
"this condition will fail. "
f"Register concerns at {report_url}"
)
mods = [
name
for name in sys.modules
if name == "distutils" or name.startswith("distutils.")
]
for name in mods:
del sys.modules[name]
def enabled():
"""
Allow selection of distutils by environment variable.
"""
which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local')
if which == 'stdlib':
import warnings
warnings.warn(
"Reliance on distutils from stdlib is deprecated. Users "
"must rely on setuptools to provide the distutils module. "
"Avoid importing distutils or import setuptools first, "
"and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. "
f"Register concerns at {report_url}"
)
return which == 'local'
def ensure_local_distutils():
import importlib
clear_distutils()
# With the DistutilsMetaFinder in place,
# perform an import to cause distutils to be
# loaded from setuptools._distutils. Ref #2906.
with shim():
importlib.import_module('distutils')
# check that submodules load as expected
core = importlib.import_module('distutils.core')
assert '_distutils' in core.__file__, core.__file__
assert 'setuptools._distutils.log' not in sys.modules
def do_override():
"""
Ensure that the local copy of distutils is preferred over stdlib.
See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
for more motivation.
"""
if enabled():
warn_distutils_present()
ensure_local_distutils()
class _TrivialRe:
def __init__(self, *patterns) -> None:
self._patterns = patterns
def match(self, string):
return all(pat in string for pat in self._patterns)
class DistutilsMetaFinder:
def find_spec(self, fullname, path, target=None):
# optimization: only consider top level modules and those
# found in the CPython test suite.
if path is not None and not fullname.startswith('test.'):
return None
method_name = 'spec_for_{fullname}'.format(**locals())
method = getattr(self, method_name, lambda: None)
return method()
def spec_for_distutils(self):
if self.is_cpython():
return None
import importlib
import importlib.abc
import importlib.util
try:
mod = importlib.import_module('setuptools._distutils')
except Exception:
# There are a couple of cases where setuptools._distutils
# may not be present:
# - An older Setuptools without a local distutils is
# taking precedence. Ref #2957.
# - Path manipulation during sitecustomize removes
# setuptools from the path but only after the hook
# has been loaded. Ref #2980.
# In either case, fall back to stdlib behavior.
return None
class DistutilsLoader(importlib.abc.Loader):
def create_module(self, spec):
mod.__name__ = 'distutils'
return mod
def exec_module(self, module):
pass
return importlib.util.spec_from_loader(
'distutils', DistutilsLoader(), origin=mod.__file__
)
@staticmethod
def is_cpython():
"""
Suppress supplying distutils for CPython (build and tests).
Ref #2965 and #3007.
"""
return os.path.isfile('pybuilddir.txt')
def spec_for_pip(self):
"""
Ensure stdlib distutils when running under pip.
See pypa/pip#8761 for rationale.
"""
if sys.version_info >= (3, 12) or self.pip_imported_during_build():
return
clear_distutils()
self.spec_for_distutils = lambda: None
@classmethod
def pip_imported_during_build(cls):
"""
Detect if pip is being imported in a build script. Ref #2355.
"""
import traceback
return any(
cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None)
)
@staticmethod
def frame_file_is_setup(frame):
"""
Return True if the indicated frame suggests a setup.py file.
"""
# some frames may not have __file__ (#2940)
return frame.f_globals.get('__file__', '').endswith('setup.py')
def spec_for_sensitive_tests(self):
"""
Ensure stdlib distutils when running select tests under CPython.
python/cpython#91169
"""
clear_distutils()
self.spec_for_distutils = lambda: None
sensitive_tests = (
[
'test.test_distutils',
'test.test_peg_generator',
'test.test_importlib',
]
if sys.version_info < (3, 10)
else [
'test.test_distutils',
]
)
for name in DistutilsMetaFinder.sensitive_tests:
setattr(
DistutilsMetaFinder,
f'spec_for_{name}',
DistutilsMetaFinder.spec_for_sensitive_tests,
)
DISTUTILS_FINDER = DistutilsMetaFinder()
def add_shim():
DISTUTILS_FINDER in sys.meta_path or insert_shim()
class shim:
def __enter__(self) -> None:
insert_shim()
def __exit__(self, exc: object, value: object, tb: object) -> None:
_remove_shim()
def insert_shim():
sys.meta_path.insert(0, DISTUTILS_FINDER)
def _remove_shim():
try:
sys.meta_path.remove(DISTUTILS_FINDER)
except ValueError:
pass
if sys.version_info < (3, 12):
# DistutilsMetaFinder can only be disabled in Python < 3.12 (PEP 632)
remove_shim = _remove_shim
@@ -0,0 +1 @@
__import__('_distutils_hack').do_override()
@@ -0,0 +1 @@
pip
@@ -0,0 +1,118 @@
Metadata-Version: 2.4
Name: distlib
Version: 0.4.3
Summary: Distribution utilities
Home-page: https://github.com/pypa/distlib
Author: Vinay Sajip
Author-email: vinay_sajip@red-dove.com
License: PSF-2.0
Project-URL: Documentation, https://distlib.readthedocs.io/
Project-URL: Source, https://github.com/pypa/distlib
Project-URL: Tracker, https://github.com/pypa/distlib/issues
Platform: any
Classifier: Development Status :: 5 - Production/Stable
Classifier: Environment :: Console
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: Python Software Foundation License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: 3.14
Classifier: Topic :: Software Development
License-File: LICENSE.txt
Dynamic: license-file
|badge1| |badge2|
.. |badge1| image:: https://img.shields.io/github/actions/workflow/status/pypa/distlib/package-tests.yml
:alt: GitHub Workflow Status (with event)
.. |badge2| image:: https://img.shields.io/codecov/c/github/pypa/distlib
:target: https://app.codecov.io/gh/pypa/distlib
:alt: GitHub coverage status
What is it?
-----------
Distlib is a library which implements low-level functions that relate to
packaging and distribution of Python software. It is intended to be used as the
basis for third-party packaging tools. The documentation is available at
https://distlib.readthedocs.io/
Main features
-------------
Distlib currently offers the following features:
* The package ``distlib.database``, which implements a database of installed
distributions, as defined by :pep:`376`, and distribution dependency graph
logic. Support is also provided for non-installed distributions (i.e.
distributions registered with metadata on an index like PyPI), including
the ability to scan for dependencies and building dependency graphs.
* The package ``distlib.index``, which implements an interface to perform
operations on an index, such as registering a project, uploading a
distribution or uploading documentation. Support is included for verifying
SSL connections (with domain matching) and signing/verifying packages using
GnuPG.
* The package ``distlib.metadata``, which implements distribution metadata as
defined by :pep:`643`, :pep:`566`, :pep:`345`, :pep:`314` and :pep:`241`.
* The package ``distlib.markers``, which implements environment markers as
defined by :pep:`508`.
* The package ``distlib.manifest``, which implements lists of files used
in packaging source distributions.
* The package ``distlib.locators``, which allows finding distributions, whether
on PyPI (XML-RPC or via the "simple" interface), local directories or some
other source.
* The package ``distlib.resources``, which allows access to data files stored
in Python packages, both in the file system and in .zip files.
* The package ``distlib.scripts``, which allows installing of scripts with
adjustment of shebang lines and support for native Windows executable
launchers.
* The package ``distlib.version``, which implements version specifiers as
defined by :pep:`440`, but also support for working with "legacy" versions and
semantic versions.
* The package ``distlib.wheel``, which provides support for building and
installing from the Wheel format for binary distributions (see :pep:`427`).
* The package ``distlib.util``, which contains miscellaneous functions and
classes which are useful in packaging, but which do not fit neatly into
one of the other packages in ``distlib``.* The package implements enhanced
globbing functionality such as the ability to use ``**`` in patterns to
specify recursing into subdirectories.
Python version and platform compatibility
-----------------------------------------
Distlib is intended to be used on and is tested on Python versions 2.7 and 3.6 or later,
pypy-2.7 and pypy3 on Linux, Windows, and macOS.
Project status
--------------
The project has reached a mature status in its development: there is a comprehensive
test suite and it has been exercised on Windows, Ubuntu and macOS. The project is used
by well-known projects such as `pip <https://pypi.org/pypi/pip>`_ and `caniusepython3
<https://pypi.org/pypi/caniusepython3>`_.
This project was migrated from Mercurial to Git and from BitBucket to GitHub, and
although all information of importance has been retained across the migration, some
commit references in issues and issue comments may have become invalid.
Code of Conduct
---------------
Everyone interacting in the distlib project's codebases, issue trackers, chat
rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_.
.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/
@@ -0,0 +1,38 @@
distlib-0.4.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
distlib-0.4.3.dist-info/METADATA,sha256=bJiPjCP6oO-HNPaGOzKXbWX1hRMzFh6KjfPYD8V-CA0,5317
distlib-0.4.3.dist-info/RECORD,,
distlib-0.4.3.dist-info/WHEEL,sha256=TdQ5LtNwLuxTCjgxN51AgdU5w-KkB9ttmLbzjTH02pg,109
distlib-0.4.3.dist-info/licenses/LICENSE.txt,sha256=gI4QyKarjesUn_mz-xn0R6gICUYG1xKpylf-rTVSWZ0,14531
distlib-0.4.3.dist-info/top_level.txt,sha256=9BERqitu_vzyeyILOcGzX9YyA2AB_xlC4-81V6xoizk,8
distlib/__init__.py,sha256=BCeUqSmmNeRltF7Jxe0rwUQL13WCaCvhmQWQWkA0kn0,625
distlib/__pycache__/__init__.cpython-311.pyc,,
distlib/__pycache__/compat.cpython-311.pyc,,
distlib/__pycache__/database.cpython-311.pyc,,
distlib/__pycache__/index.cpython-311.pyc,,
distlib/__pycache__/locators.cpython-311.pyc,,
distlib/__pycache__/manifest.cpython-311.pyc,,
distlib/__pycache__/markers.cpython-311.pyc,,
distlib/__pycache__/metadata.cpython-311.pyc,,
distlib/__pycache__/resources.cpython-311.pyc,,
distlib/__pycache__/scripts.cpython-311.pyc,,
distlib/__pycache__/util.cpython-311.pyc,,
distlib/__pycache__/version.cpython-311.pyc,,
distlib/__pycache__/wheel.cpython-311.pyc,,
distlib/compat.py,sha256=HTzhOY98c_fyMVE7xtVuZZ-AVkN4-RGes-hWmvuwquk,40605
distlib/database.py,sha256=mHy_LxiXIsIVRb-T0-idBrVLw3Ffij5teHCpbjmJ9YU,51160
distlib/index.py,sha256=lTbw268rRhj8dw1sib3VZ_0EhSGgoJO3FKJzSFMOaeA,20797
distlib/locators.py,sha256=ybXQXer-4H0xrmJm-DxsaiswG-Lw0vr1ySaoOtSZtKA,52585
distlib/manifest.py,sha256=eQpAu1qQdTQBNa8uJ-cmUjiBtORBMd8p6WcmYGBwJg8,13643
distlib/markers.py,sha256=HsgcqMkOIhKJE6vOu8j9m-SC_Np4iOtj2GXkZ0VSmBk,5269
distlib/metadata.py,sha256=eUvMnfX7Xv0jWcOkQ_FqEcp4peDTuqhayIhnOsM5XXs,39549
distlib/resources.py,sha256=_6rKusMQ4btrTzEFLXVPmT8ycKgqxbMAPNZHUYWjYXM,11508
distlib/scripts.py,sha256=rVm4A-YfGQDpsORwOy7AwepyRTayuy6rc-MnaCMBAcM,18835
distlib/t32.exe,sha256=a0GV5kCoWsMutvliiCKmIgV98eRZ33wXoS-XrqvJQVs,97792
distlib/t64-arm.exe,sha256=68TAa32V504xVBnufojh0PcenpR3U4wAqTqf-MZqbPw,182784
distlib/t64.exe,sha256=gaYY8hy4fbkHYTTnA4i26ct8IQZzkBG2pRdy0iyuBrc,108032
distlib/util.py,sha256=n2QDXvB0XtwrtehygnwdLgLkLia58sZjzgZPKyD6rAE,68167
distlib/version.py,sha256=s5VIs8wBn0fxzGxWM_aA2ZZyx525HcZbMvcTlTyZ3Rg,23727
distlib/w32.exe,sha256=R4csx3-OGM9kL4aPIzQKRo5TfmRSHZo6QWyLhDhNBks,91648
distlib/w64-arm.exe,sha256=xdyYhKj0WDcVUOCb05blQYvzdYIKMbmJn2SZvzkcey4,168448
distlib/w64.exe,sha256=ejGf-rojoBfXseGLpya6bFTFPWRG21X5KvU8J5iU-K0,101888
distlib/wheel.py,sha256=jXG3BwSMktNBS9PYVvwq-79NUfgFY3UyM2CSEQJR-1Q,45380
@@ -0,0 +1,6 @@
Wheel-Version: 1.0
Generator: setuptools (82.0.1)
Root-Is-Purelib: true
Tag: py2-none-any
Tag: py3-none-any
@@ -0,0 +1,284 @@
A. HISTORY OF THE SOFTWARE
==========================
Python was created in the early 1990s by Guido van Rossum at Stichting
Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
as a successor of a language called ABC. Guido remains Python's
principal author, although it includes many contributions from others.
In 1995, Guido continued his work on Python at the Corporation for
National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
in Reston, Virginia where he released several versions of the
software.
In May 2000, Guido and the Python core development team moved to
BeOpen.com to form the BeOpen PythonLabs team. In October of the same
year, the PythonLabs team moved to Digital Creations (now Zope
Corporation, see http://www.zope.com). In 2001, the Python Software
Foundation (PSF, see http://www.python.org/psf/) was formed, a
non-profit organization created specifically to own Python-related
Intellectual Property. Zope Corporation is a sponsoring member of
the PSF.
All Python releases are Open Source (see http://www.opensource.org for
the Open Source Definition). Historically, most, but not all, Python
releases have also been GPL-compatible; the table below summarizes
the various releases.
Release Derived Year Owner GPL-
from compatible? (1)
0.9.0 thru 1.2 1991-1995 CWI yes
1.3 thru 1.5.2 1.2 1995-1999 CNRI yes
1.6 1.5.2 2000 CNRI no
2.0 1.6 2000 BeOpen.com no
1.6.1 1.6 2001 CNRI yes (2)
2.1 2.0+1.6.1 2001 PSF no
2.0.1 2.0+1.6.1 2001 PSF yes
2.1.1 2.1+2.0.1 2001 PSF yes
2.2 2.1.1 2001 PSF yes
2.1.2 2.1.1 2002 PSF yes
2.1.3 2.1.2 2002 PSF yes
2.2.1 2.2 2002 PSF yes
2.2.2 2.2.1 2002 PSF yes
2.2.3 2.2.2 2003 PSF yes
2.3 2.2.2 2002-2003 PSF yes
2.3.1 2.3 2002-2003 PSF yes
2.3.2 2.3.1 2002-2003 PSF yes
2.3.3 2.3.2 2002-2003 PSF yes
2.3.4 2.3.3 2004 PSF yes
2.3.5 2.3.4 2005 PSF yes
2.4 2.3 2004 PSF yes
2.4.1 2.4 2005 PSF yes
2.4.2 2.4.1 2005 PSF yes
2.4.3 2.4.2 2006 PSF yes
2.4.4 2.4.3 2006 PSF yes
2.5 2.4 2006 PSF yes
2.5.1 2.5 2007 PSF yes
2.5.2 2.5.1 2008 PSF yes
2.5.3 2.5.2 2008 PSF yes
2.6 2.5 2008 PSF yes
2.6.1 2.6 2008 PSF yes
2.6.2 2.6.1 2009 PSF yes
2.6.3 2.6.2 2009 PSF yes
2.6.4 2.6.3 2009 PSF yes
2.6.5 2.6.4 2010 PSF yes
3.0 2.6 2008 PSF yes
3.0.1 3.0 2009 PSF yes
3.1 3.0.1 2009 PSF yes
3.1.1 3.1 2009 PSF yes
3.1.2 3.1 2010 PSF yes
3.2 3.1 2010 PSF yes
Footnotes:
(1) GPL-compatible doesn't mean that we're distributing Python under
the GPL. All Python licenses, unlike the GPL, let you distribute
a modified version without making your changes open source. The
GPL-compatible licenses make it possible to combine Python with
other software that is released under the GPL; the others don't.
(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
because its license has a choice of law clause. According to
CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
is "not incompatible" with the GPL.
Thanks to the many outside volunteers who have worked under Guido's
direction to make these releases possible.
B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
===============================================================
PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
--------------------------------------------
1. This LICENSE AGREEMENT is between the Python Software Foundation
("PSF"), and the Individual or Organization ("Licensee") accessing and
otherwise using this software ("Python") in source or binary form and
its associated documentation.
2. Subject to the terms and conditions of this License Agreement, PSF hereby
grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
analyze, test, perform and/or display publicly, prepare derivative works,
distribute, and otherwise use Python alone or in any derivative version,
provided, however, that PSF's License Agreement and PSF's notice of copyright,
i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
Python Software Foundation; All Rights Reserved" are retained in Python alone or
in any derivative version prepared by Licensee.
3. In the event Licensee prepares a derivative work that is based on
or incorporates Python or any part thereof, and wants to make
the derivative work available to others as provided herein, then
Licensee hereby agrees to include in any such work a brief summary of
the changes made to Python.
4. PSF is making Python available to Licensee on an "AS IS"
basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
INFRINGE ANY THIRD PARTY RIGHTS.
5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
6. This License Agreement will automatically terminate upon a material
breach of its terms and conditions.
7. Nothing in this License Agreement shall be deemed to create any
relationship of agency, partnership, or joint venture between PSF and
Licensee. This License Agreement does not grant permission to use PSF
trademarks or trade name in a trademark sense to endorse or promote
products or services of Licensee, or any third party.
8. By copying, installing or otherwise using Python, Licensee
agrees to be bound by the terms and conditions of this License
Agreement.
BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
-------------------------------------------
BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
Individual or Organization ("Licensee") accessing and otherwise using
this software in source or binary form and its associated
documentation ("the Software").
2. Subject to the terms and conditions of this BeOpen Python License
Agreement, BeOpen hereby grants Licensee a non-exclusive,
royalty-free, world-wide license to reproduce, analyze, test, perform
and/or display publicly, prepare derivative works, distribute, and
otherwise use the Software alone or in any derivative version,
provided, however, that the BeOpen Python License is retained in the
Software, alone or in any derivative version prepared by Licensee.
3. BeOpen is making the Software available to Licensee on an "AS IS"
basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
INFRINGE ANY THIRD PARTY RIGHTS.
4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
5. This License Agreement will automatically terminate upon a material
breach of its terms and conditions.
6. This License Agreement shall be governed by and interpreted in all
respects by the law of the State of California, excluding conflict of
law provisions. Nothing in this License Agreement shall be deemed to
create any relationship of agency, partnership, or joint venture
between BeOpen and Licensee. This License Agreement does not grant
permission to use BeOpen trademarks or trade names in a trademark
sense to endorse or promote products or services of Licensee, or any
third party. As an exception, the "BeOpen Python" logos available at
http://www.pythonlabs.com/logos.html may be used according to the
permissions granted on that web page.
7. By copying, installing or otherwise using the software, Licensee
agrees to be bound by the terms and conditions of this License
Agreement.
CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
---------------------------------------
1. This LICENSE AGREEMENT is between the Corporation for National
Research Initiatives, having an office at 1895 Preston White Drive,
Reston, VA 20191 ("CNRI"), and the Individual or Organization
("Licensee") accessing and otherwise using Python 1.6.1 software in
source or binary form and its associated documentation.
2. Subject to the terms and conditions of this License Agreement, CNRI
hereby grants Licensee a nonexclusive, royalty-free, world-wide
license to reproduce, analyze, test, perform and/or display publicly,
prepare derivative works, distribute, and otherwise use Python 1.6.1
alone or in any derivative version, provided, however, that CNRI's
License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
1995-2001 Corporation for National Research Initiatives; All Rights
Reserved" are retained in Python 1.6.1 alone or in any derivative
version prepared by Licensee. Alternately, in lieu of CNRI's License
Agreement, Licensee may substitute the following text (omitting the
quotes): "Python 1.6.1 is made available subject to the terms and
conditions in CNRI's License Agreement. This Agreement together with
Python 1.6.1 may be located on the Internet using the following
unique, persistent identifier (known as a handle): 1895.22/1013. This
Agreement may also be obtained from a proxy server on the Internet
using the following URL: http://hdl.handle.net/1895.22/1013".
3. In the event Licensee prepares a derivative work that is based on
or incorporates Python 1.6.1 or any part thereof, and wants to make
the derivative work available to others as provided herein, then
Licensee hereby agrees to include in any such work a brief summary of
the changes made to Python 1.6.1.
4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
INFRINGE ANY THIRD PARTY RIGHTS.
5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
6. This License Agreement will automatically terminate upon a material
breach of its terms and conditions.
7. This License Agreement shall be governed by the federal
intellectual property law of the United States, including without
limitation the federal copyright law, and, to the extent such
U.S. federal law does not apply, by the law of the Commonwealth of
Virginia, excluding Virginia's conflict of law provisions.
Notwithstanding the foregoing, with regard to derivative works based
on Python 1.6.1 that incorporate non-separable material that was
previously distributed under the GNU General Public License (GPL), the
law of the Commonwealth of Virginia shall govern this License
Agreement only as to issues arising under or with respect to
Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this
License Agreement shall be deemed to create any relationship of
agency, partnership, or joint venture between CNRI and Licensee. This
License Agreement does not grant permission to use CNRI trademarks or
trade name in a trademark sense to endorse or promote products or
services of Licensee, or any third party.
8. By clicking on the "ACCEPT" button where indicated, or by copying,
installing or otherwise using Python 1.6.1, Licensee agrees to be
bound by the terms and conditions of this License Agreement.
ACCEPT
CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
--------------------------------------------------
Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
The Netherlands. All rights reserved.
Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee is hereby granted,
provided that the above copyright notice appear in all copies and that
both that copyright notice and this permission notice appear in
supporting documentation, and that the name of Stichting Mathematisch
Centrum or CWI not be used in advertising or publicity pertaining to
distribution of the software without specific, written prior
permission.
STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
@@ -0,0 +1 @@
distlib
@@ -0,0 +1,33 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2012-2024 Vinay Sajip.
# Licensed to the Python Software Foundation under a contributor agreement.
# See LICENSE.txt and CONTRIBUTORS.txt.
#
import logging
__version__ = '0.4.3'
class DistlibException(Exception):
pass
try:
from logging import NullHandler
except ImportError: # pragma: no cover
class NullHandler(logging.Handler):
def handle(self, record):
pass
def emit(self, record):
pass
def createLock(self):
self.lock = None
logger = logging.getLogger(__name__)
logger.addHandler(NullHandler())
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+508
View File
@@ -0,0 +1,508 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013-2023 Vinay Sajip.
# Licensed to the Python Software Foundation under a contributor agreement.
# See LICENSE.txt and CONTRIBUTORS.txt.
#
import hashlib
import logging
import os
import shutil
import subprocess
import tempfile
try:
from threading import Thread
except ImportError: # pragma: no cover
from dummy_threading import Thread
from . import DistlibException
from .compat import (HTTPBasicAuthHandler, Request, HTTPPasswordMgr,
urlparse, build_opener, string_types)
from .util import zip_dir, ServerProxy
logger = logging.getLogger(__name__)
DEFAULT_INDEX = 'https://pypi.org/pypi'
DEFAULT_REALM = 'pypi'
class PackageIndex(object):
"""
This class represents a package index compatible with PyPI, the Python
Package Index.
"""
boundary = b'----------ThIs_Is_tHe_distlib_index_bouNdaRY_$'
def __init__(self, url=None):
"""
Initialise an instance.
:param url: The URL of the index. If not specified, the URL for PyPI is
used.
"""
self.url = url or DEFAULT_INDEX
self.read_configuration()
scheme, netloc, path, params, query, frag = urlparse(self.url)
if params or query or frag or scheme not in ('http', 'https'):
raise DistlibException('invalid repository: %s' % self.url)
self.password_handler = None
self.ssl_verifier = None
self.gpg = None
self.gpg_home = None
with open(os.devnull, 'w') as sink:
# Use gpg by default rather than gpg2, as gpg2 insists on
# prompting for passwords
for s in ('gpg', 'gpg2'):
try:
rc = subprocess.check_call([s, '--version'], stdout=sink,
stderr=sink)
if rc == 0:
self.gpg = s
break
except OSError:
pass
def _get_pypirc_command(self):
"""
Get the distutils command for interacting with PyPI configurations.
:return: the command.
"""
from .util import _get_pypirc_command as cmd
return cmd()
def read_configuration(self):
"""
Read the PyPI access configuration as supported by distutils. This populates
``username``, ``password``, ``realm`` and ``url`` attributes from the
configuration.
"""
from .util import _load_pypirc
cfg = _load_pypirc(self)
self.username = cfg.get('username')
self.password = cfg.get('password')
self.realm = cfg.get('realm', 'pypi')
self.url = cfg.get('repository', self.url)
def save_configuration(self):
"""
Save the PyPI access configuration. You must have set ``username`` and
``password`` attributes before calling this method.
"""
self.check_credentials()
from .util import _store_pypirc
_store_pypirc(self)
def check_credentials(self):
"""
Check that ``username`` and ``password`` have been set, and raise an
exception if not.
"""
if self.username is None or self.password is None:
raise DistlibException('username and password must be set')
pm = HTTPPasswordMgr()
_, netloc, _, _, _, _ = urlparse(self.url)
pm.add_password(self.realm, netloc, self.username, self.password)
self.password_handler = HTTPBasicAuthHandler(pm)
def register(self, metadata): # pragma: no cover
"""
Register a distribution on PyPI, using the provided metadata.
:param metadata: A :class:`Metadata` instance defining at least a name
and version number for the distribution to be
registered.
:return: The HTTP response received from PyPI upon submission of the
request.
"""
self.check_credentials()
metadata.validate()
d = metadata.todict()
d[':action'] = 'verify'
request = self.encode_request(d.items(), [])
self.send_request(request)
d[':action'] = 'submit'
request = self.encode_request(d.items(), [])
return self.send_request(request)
def _reader(self, name, stream, outbuf):
"""
Thread runner for reading lines of from a subprocess into a buffer.
:param name: The logical name of the stream (used for logging only).
:param stream: The stream to read from. This will typically a pipe
connected to the output stream of a subprocess.
:param outbuf: The list to append the read lines to.
"""
while True:
s = stream.readline()
if not s:
break
s = s.decode('utf-8').rstrip()
outbuf.append(s)
logger.debug('%s: %s' % (name, s))
stream.close()
def get_sign_command(self, filename, signer, sign_password, keystore=None): # pragma: no cover
"""
Return a suitable command for signing a file.
:param filename: The pathname to the file to be signed.
:param signer: The identifier of the signer of the file.
:param sign_password: The passphrase for the signer's
private key used for signing.
:param keystore: The path to a directory which contains the keys
used in verification. If not specified, the
instance's ``gpg_home`` attribute is used instead.
:return: The signing command as a list suitable to be
passed to :class:`subprocess.Popen`.
"""
cmd = [self.gpg, '--status-fd', '2', '--no-tty']
if keystore is None:
keystore = self.gpg_home
if keystore:
cmd.extend(['--homedir', keystore])
if sign_password is not None:
cmd.extend(['--batch', '--passphrase-fd', '0'])
td = tempfile.mkdtemp()
sf = os.path.join(td, os.path.basename(filename) + '.asc')
cmd.extend(['--detach-sign', '--armor', '--local-user',
signer, '--output', sf, filename])
logger.debug('invoking: %s', ' '.join(cmd))
return cmd, sf
def run_command(self, cmd, input_data=None):
"""
Run a command in a child process , passing it any input data specified.
:param cmd: The command to run.
:param input_data: If specified, this must be a byte string containing
data to be sent to the child process.
:return: A tuple consisting of the subprocess' exit code, a list of
lines read from the subprocess' ``stdout``, and a list of
lines read from the subprocess' ``stderr``.
"""
kwargs = {
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE,
}
if input_data is not None:
kwargs['stdin'] = subprocess.PIPE
stdout = []
stderr = []
p = subprocess.Popen(cmd, **kwargs)
# We don't use communicate() here because we may need to
# get clever with interacting with the command
t1 = Thread(target=self._reader, args=('stdout', p.stdout, stdout))
t1.start()
t2 = Thread(target=self._reader, args=('stderr', p.stderr, stderr))
t2.start()
if input_data is not None:
p.stdin.write(input_data)
p.stdin.close()
p.wait()
t1.join()
t2.join()
return p.returncode, stdout, stderr
def sign_file(self, filename, signer, sign_password, keystore=None): # pragma: no cover
"""
Sign a file.
:param filename: The pathname to the file to be signed.
:param signer: The identifier of the signer of the file.
:param sign_password: The passphrase for the signer's
private key used for signing.
:param keystore: The path to a directory which contains the keys
used in signing. If not specified, the instance's
``gpg_home`` attribute is used instead.
:return: The absolute pathname of the file where the signature is
stored.
"""
cmd, sig_file = self.get_sign_command(filename, signer, sign_password,
keystore)
rc, stdout, stderr = self.run_command(cmd,
sign_password.encode('utf-8'))
if rc != 0:
raise DistlibException('sign command failed with error '
'code %s' % rc)
return sig_file
def upload_file(self, metadata, filename, signer=None, sign_password=None,
filetype='sdist', pyversion='source', keystore=None):
"""
Upload a release file to the index.
:param metadata: A :class:`Metadata` instance defining at least a name
and version number for the file to be uploaded.
:param filename: The pathname of the file to be uploaded.
:param signer: The identifier of the signer of the file.
:param sign_password: The passphrase for the signer's
private key used for signing.
:param filetype: The type of the file being uploaded. This is the
distutils command which produced that file, e.g.
``sdist`` or ``bdist_wheel``.
:param pyversion: The version of Python which the release relates
to. For code compatible with any Python, this would
be ``source``, otherwise it would be e.g. ``3.2``.
:param keystore: The path to a directory which contains the keys
used in signing. If not specified, the instance's
``gpg_home`` attribute is used instead.
:return: The HTTP response received from PyPI upon submission of the
request.
"""
self.check_credentials()
if not os.path.exists(filename):
raise DistlibException('not found: %s' % filename)
metadata.validate()
d = metadata.todict()
sig_file = None
if signer:
if not self.gpg:
logger.warning('no signing program available - not signed')
else:
sig_file = self.sign_file(filename, signer, sign_password,
keystore)
with open(filename, 'rb') as f:
file_data = f.read()
md5_digest = hashlib.md5(file_data).hexdigest()
sha256_digest = hashlib.sha256(file_data).hexdigest()
d.update({
':action': 'file_upload',
'protocol_version': '1',
'filetype': filetype,
'pyversion': pyversion,
'md5_digest': md5_digest,
'sha256_digest': sha256_digest,
})
files = [('content', os.path.basename(filename), file_data)]
if sig_file:
with open(sig_file, 'rb') as f:
sig_data = f.read()
files.append(('gpg_signature', os.path.basename(sig_file),
sig_data))
shutil.rmtree(os.path.dirname(sig_file))
request = self.encode_request(d.items(), files)
return self.send_request(request)
def upload_documentation(self, metadata, doc_dir): # pragma: no cover
"""
Upload documentation to the index.
:param metadata: A :class:`Metadata` instance defining at least a name
and version number for the documentation to be
uploaded.
:param doc_dir: The pathname of the directory which contains the
documentation. This should be the directory that
contains the ``index.html`` for the documentation.
:return: The HTTP response received from PyPI upon submission of the
request.
"""
self.check_credentials()
if not os.path.isdir(doc_dir):
raise DistlibException('not a directory: %r' % doc_dir)
fn = os.path.join(doc_dir, 'index.html')
if not os.path.exists(fn):
raise DistlibException('not found: %r' % fn)
metadata.validate()
name, version = metadata.name, metadata.version
zip_data = zip_dir(doc_dir).getvalue()
fields = [(':action', 'doc_upload'),
('name', name), ('version', version)]
files = [('content', name, zip_data)]
request = self.encode_request(fields, files)
return self.send_request(request)
def get_verify_command(self, signature_filename, data_filename,
keystore=None):
"""
Return a suitable command for verifying a file.
:param signature_filename: The pathname to the file containing the
signature.
:param data_filename: The pathname to the file containing the
signed data.
:param keystore: The path to a directory which contains the keys
used in verification. If not specified, the
instance's ``gpg_home`` attribute is used instead.
:return: The verifying command as a list suitable to be
passed to :class:`subprocess.Popen`.
"""
cmd = [self.gpg, '--status-fd', '2', '--no-tty']
if keystore is None:
keystore = self.gpg_home
if keystore:
cmd.extend(['--homedir', keystore])
cmd.extend(['--verify', signature_filename, data_filename])
logger.debug('invoking: %s', ' '.join(cmd))
return cmd
def verify_signature(self, signature_filename, data_filename,
keystore=None):
"""
Verify a signature for a file.
:param signature_filename: The pathname to the file containing the
signature.
:param data_filename: The pathname to the file containing the
signed data.
:param keystore: The path to a directory which contains the keys
used in verification. If not specified, the
instance's ``gpg_home`` attribute is used instead.
:return: True if the signature was verified, else False.
"""
if not self.gpg:
raise DistlibException('verification unavailable because gpg '
'unavailable')
cmd = self.get_verify_command(signature_filename, data_filename,
keystore)
rc, stdout, stderr = self.run_command(cmd)
if rc not in (0, 1):
raise DistlibException('verify command failed with error code %s' % rc)
return rc == 0
def download_file(self, url, destfile, digest=None, reporthook=None):
"""
This is a convenience method for downloading a file from an URL.
Normally, this will be a file from the index, though currently
no check is made for this (i.e. a file can be downloaded from
anywhere).
The method is just like the :func:`urlretrieve` function in the
standard library, except that it allows digest computation to be
done during download and checking that the downloaded data
matched any expected value.
:param url: The URL of the file to be downloaded (assumed to be
available via an HTTP GET request).
:param destfile: The pathname where the downloaded file is to be
saved.
:param digest: If specified, this must be a (hasher, value)
tuple, where hasher is the algorithm used (e.g.
``'md5'``) and ``value`` is the expected value.
:param reporthook: The same as for :func:`urlretrieve` in the
standard library.
"""
if digest is None:
digester = None
logger.debug('No digest specified')
else:
if isinstance(digest, (list, tuple)):
hasher, digest = digest
else:
hasher = 'md5'
digester = getattr(hashlib, hasher)()
logger.debug('Digest specified: %s' % digest)
# The following code is equivalent to urlretrieve.
# We need to do it this way so that we can compute the
# digest of the file as we go.
with open(destfile, 'wb') as dfp:
# addinfourl is not a context manager on 2.x
# so we have to use try/finally
sfp = self.send_request(Request(url))
try:
headers = sfp.info()
blocksize = 8192
size = -1
read = 0
blocknum = 0
if "content-length" in headers:
size = int(headers["Content-Length"])
if reporthook:
reporthook(blocknum, blocksize, size)
while True:
block = sfp.read(blocksize)
if not block:
break
read += len(block)
dfp.write(block)
if digester:
digester.update(block)
blocknum += 1
if reporthook:
reporthook(blocknum, blocksize, size)
finally:
sfp.close()
# check that we got the whole file, if we can
if size >= 0 and read < size:
raise DistlibException(
'retrieval incomplete: got only %d out of %d bytes'
% (read, size))
# if we have a digest, it must match.
if digester:
actual = digester.hexdigest()
if digest != actual:
raise DistlibException('%s digest mismatch for %s: expected '
'%s, got %s' % (hasher, destfile,
digest, actual))
logger.debug('Digest verified: %s', digest)
def send_request(self, req):
"""
Send a standard library :class:`Request` to PyPI and return its
response.
:param req: The request to send.
:return: The HTTP response from PyPI (a standard library HTTPResponse).
"""
handlers = []
if self.password_handler:
handlers.append(self.password_handler)
if self.ssl_verifier:
handlers.append(self.ssl_verifier)
opener = build_opener(*handlers)
return opener.open(req)
def encode_request(self, fields, files):
"""
Encode fields and files for posting to an HTTP server.
:param fields: The fields to send as a list of (fieldname, value)
tuples.
:param files: The files to send as a list of (fieldname, filename,
file_bytes) tuple.
"""
# Adapted from packaging, which in turn was adapted from
# http://code.activestate.com/recipes/146306
parts = []
boundary = self.boundary
for k, values in fields:
if not isinstance(values, (list, tuple)):
values = [values]
for v in values:
parts.extend((
b'--' + boundary,
('Content-Disposition: form-data; name="%s"' %
k).encode('utf-8'),
b'',
v.encode('utf-8')))
for key, filename, value in files:
parts.extend((
b'--' + boundary,
('Content-Disposition: form-data; name="%s"; filename="%s"' %
(key, filename)).encode('utf-8'),
b'',
value))
parts.extend((b'--' + boundary + b'--', b''))
body = b'\r\n'.join(parts)
ct = b'multipart/form-data; boundary=' + boundary
headers = {
'Content-type': ct,
'Content-length': str(len(body))
}
return Request(self.url, body, headers)
def search(self, terms, operator=None): # pragma: no cover
if isinstance(terms, string_types):
terms = {'name': terms}
rpc_proxy = ServerProxy(self.url, timeout=3.0)
try:
return rpc_proxy.search(terms, operator or 'and')
finally:
rpc_proxy('close')()
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,367 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2012-2026 Python Software Foundation.
# See LICENSE.txt and CONTRIBUTORS.txt.
#
"""
Class representing the list of files in a distribution.
Equivalent to distutils.filelist, but fixes some problems.
"""
import fnmatch
import logging
import os
import re
import sys
from . import DistlibException
from .compat import fsdecode
from .util import convert_path
__all__ = ['Manifest']
logger = logging.getLogger(__name__)
# a \ followed by some spaces + EOL
_COLLAPSE_PATTERN = re.compile('\\\\w*\n', re.M)
_COMMENTED_LINE = re.compile('#.*?(?=\n)|\n(?=$)', re.M | re.S)
#
# Due to the different results returned by fnmatch.translate, we need
# to do slightly different processing for Python 2.7 and 3.2 ... this needed
# to be brought in for Python 3.6 onwards.
#
_PYTHON_VERSION = sys.version_info[:2]
class Manifest(object):
"""
A list of files built by exploring the filesystem and filtered by applying various
patterns to what we find there.
"""
def __init__(self, base=None):
"""
Initialise an instance.
:param base: The base directory to explore under.
"""
self.base = os.path.abspath(os.path.normpath(base or os.getcwd()))
self.prefix = self.base + os.sep
self.allfiles = None
self.files = set()
#
# Public API
#
def findall(self):
"""Find all files under the base and set ``allfiles`` to the absolute
pathnames of files found.
"""
from stat import S_ISREG, S_ISDIR, S_ISLNK
self.allfiles = allfiles = []
root = self.base
stack = [root]
pop = stack.pop
push = stack.append
while stack:
root = pop()
names = os.listdir(root)
for name in names:
fullname = os.path.join(root, name)
# Avoid excess stat calls -- just one will do, thank you!
stat = os.lstat(fullname)
mode = stat.st_mode
if S_ISREG(mode):
allfiles.append(fsdecode(fullname))
elif S_ISDIR(mode) and not S_ISLNK(mode):
push(fullname)
def add(self, item):
"""
Add a file to the manifest.
:param item: The pathname to add. This can be relative to the base.
"""
if not item.startswith(self.prefix):
item = os.path.join(self.base, item)
self.files.add(os.path.normpath(item))
def add_many(self, items):
"""
Add a list of files to the manifest.
:param items: The pathnames to add. These can be relative to the base.
"""
for item in items:
self.add(item)
def sorted(self, wantdirs=False):
"""
Return sorted files in directory order
"""
def add_dir(dirs, d):
dirs.add(d)
logger.debug('add_dir added %s', d)
if d != self.base:
parent, _ = os.path.split(d)
assert parent not in ('', '/')
add_dir(dirs, parent)
result = set(self.files) # make a copy!
if wantdirs:
dirs = set()
for f in result:
add_dir(dirs, os.path.dirname(f))
result |= dirs
return [os.path.join(*path_tuple) for path_tuple in sorted(os.path.split(path) for path in result)]
def clear(self):
"""Clear all collected files."""
self.files = set()
self.allfiles = []
def process_directive(self, directive):
"""
Process a directive which either adds some files from ``allfiles`` to
``files``, or removes some files from ``files``.
:param directive: The directive to process. This should be in a format
compatible with distutils ``MANIFEST.in`` files:
http://docs.python.org/distutils/sourcedist.html#commands
"""
# Parse the line: split it up, make sure the right number of words
# is there, and return the relevant words. 'action' is always
# defined: it's the first word of the line. Which of the other
# three are defined depends on the action; it'll be either
# patterns, (dir and patterns), or (dirpattern).
action, patterns, thedir, dirpattern = self._parse_directive(directive)
# OK, now we know that the action is valid and we have the
# right number of words on the line for that action -- so we
# can proceed with minimal error-checking.
if action == 'include':
for pattern in patterns:
if not self._include_pattern(pattern, anchor=True):
logger.warning('no files found matching %r', pattern)
elif action == 'exclude':
for pattern in patterns:
self._exclude_pattern(pattern, anchor=True)
elif action == 'global-include':
for pattern in patterns:
if not self._include_pattern(pattern, anchor=False):
logger.warning('no files found matching %r '
'anywhere in distribution', pattern)
elif action == 'global-exclude':
for pattern in patterns:
self._exclude_pattern(pattern, anchor=False)
elif action == 'recursive-include':
for pattern in patterns:
if not self._include_pattern(pattern, prefix=thedir):
logger.warning('no files found matching %r '
'under directory %r', pattern, thedir)
elif action == 'recursive-exclude':
for pattern in patterns:
self._exclude_pattern(pattern, prefix=thedir)
elif action == 'graft':
if not self._include_pattern(None, prefix=dirpattern):
logger.warning('no directories found matching %r', dirpattern)
elif action == 'prune':
if not self._exclude_pattern(None, prefix=dirpattern):
logger.warning('no previously-included directories found '
'matching %r', dirpattern)
else: # pragma: no cover
# This should never happen, as it should be caught in
# _parse_template_line
raise DistlibException('invalid action %r' % action)
#
# Private API
#
def _parse_directive(self, directive):
"""
Validate a directive.
:param directive: The directive to validate.
:return: A tuple of action, patterns, thedir, dir_patterns
"""
words = directive.split()
if len(words) == 1 and words[0] not in ('include', 'exclude', 'global-include', 'global-exclude',
'recursive-include', 'recursive-exclude', 'graft', 'prune'):
# no action given, let's use the default 'include'
words.insert(0, 'include')
action = words[0]
patterns = thedir = dir_pattern = None
if action in ('include', 'exclude', 'global-include', 'global-exclude'):
if len(words) < 2:
raise DistlibException('%r expects <pattern1> <pattern2> ...' % action)
patterns = [convert_path(word) for word in words[1:]]
elif action in ('recursive-include', 'recursive-exclude'):
if len(words) < 3:
raise DistlibException('%r expects <dir> <pattern1> <pattern2> ...' % action)
thedir = convert_path(words[1])
patterns = [convert_path(word) for word in words[2:]]
elif action in ('graft', 'prune'):
if len(words) != 2:
raise DistlibException('%r expects a single <dir_pattern>' % action)
dir_pattern = convert_path(words[1])
else:
raise DistlibException('unknown action %r' % action)
return action, patterns, thedir, dir_pattern
def _include_pattern(self, pattern, anchor=True, prefix=None, is_regex=False):
"""Select strings (presumably filenames) from 'self.files' that
match 'pattern', a Unix-style wildcard (glob) pattern.
Patterns are not quite the same as implemented by the 'fnmatch'
module: '*' and '?' match non-special characters, where "special"
is platform-dependent: slash on Unix; colon, slash, and backslash on
DOS/Windows; and colon on Mac OS.
If 'anchor' is true (the default), then the pattern match is more
stringent: "*.py" will match "foo.py" but not "foo/bar.py". If
'anchor' is false, both of these will match.
If 'prefix' is supplied, then only filenames starting with 'prefix'
(itself a pattern) and ending with 'pattern', with anything in between
them, will match. 'anchor' is ignored in this case.
If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
'pattern' is assumed to be either a string containing a regex or a
regex object -- no translation is done, the regex is just compiled
and used as-is.
Selected strings will be added to self.files.
Return True if files are found.
"""
# XXX docstring lying about what the special chars are?
found = False
pattern_re = self._translate_pattern(pattern, anchor, prefix, is_regex)
# delayed loading of allfiles list
if self.allfiles is None:
self.findall()
for name in self.allfiles:
if pattern_re.search(name):
self.files.add(name)
found = True
return found
def _exclude_pattern(self, pattern, anchor=True, prefix=None, is_regex=False):
"""Remove strings (presumably filenames) from 'files' that match
'pattern'.
Other parameters are the same as for 'include_pattern()', above.
The list 'self.files' is modified in place. Return True if files are
found.
This API is public to allow e.g. exclusion of SCM subdirs, e.g. when
packaging source distributions
"""
found = False
pattern_re = self._translate_pattern(pattern, anchor, prefix, is_regex)
for f in list(self.files):
if pattern_re.search(f):
self.files.remove(f)
found = True
return found
def _translate_pattern(self, pattern, anchor=True, prefix=None, is_regex=False):
"""Translate a shell-like wildcard pattern to a compiled regular
expression.
Return the compiled regex. If 'is_regex' true,
then 'pattern' is directly compiled to a regex (if it's a string)
or just returned as-is (assumes it's a regex object).
"""
if is_regex:
if isinstance(pattern, str):
return re.compile(pattern)
else:
return pattern
if _PYTHON_VERSION > (3, 2):
# ditch start and end characters
start, _, end = self._glob_to_re('_').partition('_')
if pattern:
pattern_re = self._glob_to_re(pattern)
if _PYTHON_VERSION > (3, 2):
assert pattern_re.startswith(start) and pattern_re.endswith(end)
else:
pattern_re = ''
base = re.escape(os.path.join(self.base, ''))
if prefix is not None:
# ditch end of pattern character
if _PYTHON_VERSION <= (3, 2):
empty_pattern = self._glob_to_re('')
prefix_re = self._glob_to_re(prefix)[:-len(empty_pattern)]
else:
prefix_re = self._glob_to_re(prefix)
assert prefix_re.startswith(start) and prefix_re.endswith(end)
prefix_re = prefix_re[len(start):len(prefix_re) - len(end)]
sep = os.sep
if os.sep == '\\':
sep = r'\\'
if _PYTHON_VERSION <= (3, 2):
pattern_re = '^' + base + sep.join((prefix_re, '.*' + pattern_re))
else:
pattern_re = pattern_re[len(start):len(pattern_re) - len(end)]
pattern_re = r'%s%s%s%s.*%s%s' % (start, base, prefix_re, sep, pattern_re, end)
else: # no prefix -- respect anchor flag
if anchor:
if _PYTHON_VERSION <= (3, 2):
pattern_re = '^' + base + pattern_re
else:
pattern_re = r'%s%s%s' % (start, base, pattern_re[len(start):])
return re.compile(pattern_re)
def _glob_to_re(self, pattern):
"""Translate a shell-like glob pattern to a regular expression.
Return a string containing the regex. Differs from
'fnmatch.translate()' in that '*' does not match "special characters"
(which are platform-specific).
"""
pattern_re = fnmatch.translate(pattern)
# '?' and '*' in the glob pattern become '.' and '.*' in the RE, which
# IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
# and by extension they shouldn't match such "special characters" under
# any OS. So change all non-escaped dots in the RE to match any
# character except the special characters (currently: just os.sep).
sep = os.sep
if os.sep == '\\':
# we're using a regex to manipulate a regex, so we need
# to escape the backslash twice
sep = r'\\\\'
escaped = r'\1[^%s]' % sep
pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
return pattern_re
@@ -0,0 +1,164 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2012-2023 Vinay Sajip.
# Licensed to the Python Software Foundation under a contributor agreement.
# See LICENSE.txt and CONTRIBUTORS.txt.
#
"""
Parser for the environment markers micro-language defined in PEP 508.
"""
# Note: In PEP 345, the micro-language was Python compatible, so the ast
# module could be used to parse it. However, PEP 508 introduced operators such
# as ~= and === which aren't in Python, necessitating a different approach.
import os
import re
import sys
import platform
from .compat import string_types
from .util import in_venv, parse_marker
from .version import LegacyVersion as LV
__all__ = ['interpret']
_VERSION_PATTERN = re.compile(r'((\d+(\.\d+)*\w*)|\'(\d+(\.\d+)*\w*)\'|\"(\d+(\.\d+)*\w*)\")')
_VERSION_MARKERS = {'python_version', 'python_full_version'}
def _is_version_marker(s):
return isinstance(s, string_types) and s in _VERSION_MARKERS
def _is_literal(o):
if not isinstance(o, string_types) or not o:
return False
return o[0] in '\'"'
def _get_versions(s):
return {LV(m.groups()[0]) for m in _VERSION_PATTERN.finditer(s)}
class Evaluator(object):
"""
This class is used to evaluate marker expressions.
"""
operations = {
'==': lambda x, y: x == y,
'===': lambda x, y: x == y,
'~=': lambda x, y: x == y or x > y,
'!=': lambda x, y: x != y,
'<': lambda x, y: x < y,
'<=': lambda x, y: x == y or x < y,
'>': lambda x, y: x > y,
'>=': lambda x, y: x == y or x > y,
'and': lambda x, y: x and y,
'or': lambda x, y: x or y,
'in': lambda x, y: x in y,
'not in': lambda x, y: x not in y,
}
def evaluate(self, expr, context):
"""
Evaluate a marker expression returned by the :func:`parse_requirement`
function in the specified context.
"""
if isinstance(expr, string_types):
if expr[0] in '\'"':
result = expr[1:-1]
else:
if expr not in context:
raise SyntaxError('unknown variable: %s' % expr)
result = context[expr]
else:
assert isinstance(expr, dict)
op = expr['op']
if op not in self.operations:
raise NotImplementedError('op not implemented: %s' % op)
elhs = expr['lhs']
erhs = expr['rhs']
if _is_literal(expr['lhs']) and _is_literal(expr['rhs']):
raise SyntaxError('invalid comparison: %s %s %s' % (elhs, op, erhs))
lhs = self.evaluate(elhs, context)
rhs = self.evaluate(erhs, context)
if ((_is_version_marker(elhs) or _is_version_marker(erhs)) and
op in ('<', '<=', '>', '>=', '===', '==', '!=', '~=')):
lhs = LV(lhs)
rhs = LV(rhs)
elif _is_version_marker(elhs) and op in ('in', 'not in'):
lhs = LV(lhs)
rhs = _get_versions(rhs)
result = self.operations[op](lhs, rhs)
return result
_DIGITS = re.compile(r'\d+\.\d+')
def default_context():
def format_full_version(info):
version = '%s.%s.%s' % (info.major, info.minor, info.micro)
kind = info.releaselevel
if kind != 'final':
version += kind[0] + str(info.serial)
return version
if hasattr(sys, 'implementation'):
implementation_version = format_full_version(sys.implementation.version)
implementation_name = sys.implementation.name
else:
implementation_version = '0'
implementation_name = ''
ppv = platform.python_version()
m = _DIGITS.match(ppv)
pv = m.group(0)
result = {
'implementation_name': implementation_name,
'implementation_version': implementation_version,
'os_name': os.name,
'platform_machine': platform.machine(),
'platform_python_implementation': platform.python_implementation(),
'platform_release': platform.release(),
'platform_system': platform.system(),
'platform_version': platform.version(),
'platform_in_venv': str(in_venv()),
'python_full_version': ppv,
'python_version': pv,
'sys_platform': sys.platform,
}
return result
DEFAULT_CONTEXT = default_context()
del default_context
evaluator = Evaluator()
def interpret_parsed(expr, execution_context=None):
context = dict(DEFAULT_CONTEXT)
if execution_context:
context.update(execution_context)
return evaluator.evaluate(expr, context)
def interpret(marker, execution_context=None):
"""
Interpret a marker and return a result depending on environment.
:param marker: The marker to interpret.
:type marker: str
:param execution_context: The context used for name lookup.
:type execution_context: mapping
"""
try:
expr, rest = parse_marker(marker)
except Exception as e:
raise SyntaxError('Unable to interpret marker syntax: %s: %s' % (marker, e))
if rest and rest[0] != '#':
raise SyntaxError('unexpected trailing data in marker: %s: %s' % (marker, rest))
return interpret_parsed(expr, execution_context)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,374 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013-2026 Vinay Sajip.
# Licensed to the Python Software Foundation under a contributor agreement.
# See LICENSE.txt and CONTRIBUTORS.txt.
#
from __future__ import unicode_literals
import bisect
import io
import logging
import os
import pkgutil
import sys
import types
import zipimport
from . import DistlibException
from .util import cached_property, get_cache_base, Cache
logger = logging.getLogger(__name__)
cache = None # created when needed
class ResourceCache(Cache):
def __init__(self, base=None):
if base is None:
# Use native string to avoid issues on 2.x: see Python #20140.
base = os.path.join(get_cache_base(), str('resource-cache'))
super(ResourceCache, self).__init__(base)
def is_stale(self, resource, path):
"""
Is the cache stale for the given resource?
:param resource: The :class:`Resource` being cached.
:param path: The path of the resource in the cache.
:return: True if the cache is stale.
"""
# Cache invalidation is a hard problem :-)
return True
def get(self, resource):
"""
Get a resource into the cache,
:param resource: A :class:`Resource` instance.
:return: The pathname of the resource in the cache.
"""
prefix, path = resource.finder.get_cache_info(resource)
if prefix is None:
result = path
else:
result = os.path.join(self.base, self.prefix_to_dir(prefix), path)
dirname = os.path.dirname(result)
if not os.path.isdir(dirname):
os.makedirs(dirname)
if not os.path.exists(result):
stale = True
else:
stale = self.is_stale(resource, path)
if stale:
# write the bytes of the resource to the cache location
with open(result, 'wb') as f:
f.write(resource.bytes)
return result
class ResourceBase(object):
def __init__(self, finder, name):
self.finder = finder
self.name = name
class Resource(ResourceBase):
"""
A class representing an in-package resource, such as a data file. This is
not normally instantiated by user code, but rather by a
:class:`ResourceFinder` which manages the resource.
"""
is_container = False # Backwards compatibility
def as_stream(self):
"""
Get the resource as a stream.
This is not a property to make it obvious that it returns a new stream
each time.
"""
return self.finder.get_stream(self)
@cached_property
def file_path(self):
global cache
if cache is None:
cache = ResourceCache()
return cache.get(self)
@cached_property
def bytes(self):
return self.finder.get_bytes(self)
@cached_property
def size(self):
return self.finder.get_size(self)
class ResourceContainer(ResourceBase):
is_container = True # Backwards compatibility
@cached_property
def resources(self):
return self.finder.get_resources(self)
class ResourceFinder(object):
"""
Resource finder for file system resources.
"""
if sys.platform.startswith('java'):
skipped_extensions = ('.pyc', '.pyo', '.class')
else:
skipped_extensions = ('.pyc', '.pyo')
def __init__(self, module):
self.module = module
self.loader = getattr(module, '__loader__', None)
self.base = os.path.dirname(getattr(module, '__file__', ''))
def _adjust_path(self, path):
return os.path.realpath(path)
def _is_in_base(self, path):
base = self._adjust_path(self.base)
if path == base:
return True
if not base.endswith(os.sep):
base = base + os.sep
return path.startswith(base)
def _make_path(self, resource_name):
# Issue #50: need to preserve type of path on Python 2.x
# like os.path._get_sep
if isinstance(resource_name, bytes): # should only happen on 2.x
sep = b'/'
else:
sep = '/'
parts = resource_name.split(sep)
parts.insert(0, self.base)
result = os.path.join(*parts)
result = self._adjust_path(result)
# See issue 263: code below commented out as it proved to be
# too aggressively cautious, causing problems in valid use case scenarios.
# Confine the resolved resource to the package base so a resource
# name containing '..' cannot read files outside the package.
# if not self._is_in_base(result):
# raise DistlibException('Resource name escapes package: '
# '%r' % resource_name)
return result
def _find(self, path):
return os.path.exists(path)
def get_cache_info(self, resource):
return None, resource.path
def find(self, resource_name):
path = self._make_path(resource_name)
if not self._find(path):
result = None
else:
if self._is_directory(path):
result = ResourceContainer(self, resource_name)
else:
result = Resource(self, resource_name)
result.path = path
return result
def get_stream(self, resource):
return open(resource.path, 'rb')
def get_bytes(self, resource):
with open(resource.path, 'rb') as f:
return f.read()
def get_size(self, resource):
return os.path.getsize(resource.path)
def get_resources(self, resource):
def allowed(f):
return (f != '__pycache__' and not f.endswith(self.skipped_extensions))
return set([f for f in os.listdir(resource.path) if allowed(f)])
def is_container(self, resource):
return self._is_directory(resource.path)
_is_directory = staticmethod(os.path.isdir)
def iterator(self, resource_name):
resource = self.find(resource_name)
if resource is not None:
todo = [resource]
while todo:
resource = todo.pop(0)
yield resource
if resource.is_container:
rname = resource.name
for name in resource.resources:
if not rname:
new_name = name
else:
new_name = '/'.join([rname, name])
child = self.find(new_name)
if child.is_container:
todo.append(child)
else:
yield child
class ZipResourceFinder(ResourceFinder):
"""
Resource finder for resources in .zip files.
"""
def __init__(self, module):
super(ZipResourceFinder, self).__init__(module)
archive = self.loader.archive
self.prefix_len = 1 + len(archive)
# PyPy doesn't have a _files attr on zipimporter, and you can't set one
if hasattr(self.loader, '_files'):
self._files = self.loader._files
else:
self._files = zipimport._zip_directory_cache[archive]
self.index = sorted(self._files)
def _adjust_path(self, path):
return path
def _find(self, path):
path = path[self.prefix_len:]
if path in self._files:
result = True
else:
if path and path[-1] != os.sep:
path = path + os.sep
i = bisect.bisect(self.index, path)
try:
result = self.index[i].startswith(path)
except IndexError:
result = False
if not result:
logger.debug('_find failed: %r %r', path, self.loader.prefix)
else:
logger.debug('_find worked: %r %r', path, self.loader.prefix)
return result
def get_cache_info(self, resource):
prefix = self.loader.archive
path = resource.path[1 + len(prefix):]
return prefix, path
def get_bytes(self, resource):
return self.loader.get_data(resource.path)
def get_stream(self, resource):
return io.BytesIO(self.get_bytes(resource))
def get_size(self, resource):
path = resource.path[self.prefix_len:]
return self._files[path][3]
def get_resources(self, resource):
path = resource.path[self.prefix_len:]
if path and path[-1] != os.sep:
path += os.sep
plen = len(path)
result = set()
i = bisect.bisect(self.index, path)
while i < len(self.index):
if not self.index[i].startswith(path):
break
s = self.index[i][plen:]
result.add(s.split(os.sep, 1)[0]) # only immediate children
i += 1
return result
def _is_directory(self, path):
path = path[self.prefix_len:]
if path and path[-1] != os.sep:
path += os.sep
i = bisect.bisect(self.index, path)
try:
result = self.index[i].startswith(path)
except IndexError:
result = False
return result
_finder_registry = {type(None): ResourceFinder, zipimport.zipimporter: ZipResourceFinder}
try:
# In Python 3.6, _frozen_importlib -> _frozen_importlib_external
try:
import _frozen_importlib_external as _fi
except ImportError:
import _frozen_importlib as _fi
_finder_registry[_fi.SourceFileLoader] = ResourceFinder
_finder_registry[_fi.FileFinder] = ResourceFinder
# See issue #146
_finder_registry[_fi.SourcelessFileLoader] = ResourceFinder
del _fi
except (ImportError, AttributeError):
pass
def register_finder(loader, finder_maker):
_finder_registry[type(loader)] = finder_maker
_finder_cache = {}
def finder(package):
"""
Return a resource finder for a package.
:param package: The name of the package.
:return: A :class:`ResourceFinder` instance for the package.
"""
if package in _finder_cache:
result = _finder_cache[package]
else:
if package not in sys.modules:
__import__(package)
module = sys.modules[package]
path = getattr(module, '__path__', None)
if path is None:
raise DistlibException('You cannot get a finder for a module, '
'only for a package')
loader = getattr(module, '__loader__', None)
finder_maker = _finder_registry.get(type(loader))
if finder_maker is None:
raise DistlibException('Unable to locate finder for %r' % package)
result = finder_maker(module)
_finder_cache[package] = result
return result
_dummy_module = types.ModuleType(str('__dummy__'))
def finder_for_path(path):
"""
Return a resource finder for a path, which should represent a container.
:param path: The path.
:return: A :class:`ResourceFinder` instance for the path.
"""
result = None
# calls any path hooks, gets importer into cache
pkgutil.get_importer(path)
loader = sys.path_importer_cache.get(path)
finder = _finder_registry.get(type(loader))
if finder:
module = _dummy_module
module.__file__ = os.path.join(path, '')
module.__loader__ = loader
result = finder(module)
return result
@@ -0,0 +1,451 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013-2026 Vinay Sajip.
# Licensed to the Python Software Foundation under a contributor agreement.
# See LICENSE.txt and CONTRIBUTORS.txt.
#
from io import BytesIO
import logging
import os
import re
import struct
import sys
import time
from zipfile import ZipInfo
from . import DistlibException
from .compat import sysconfig, detect_encoding, ZipFile
from .resources import finder
from .util import (FileOperator, get_export_entry, convert_path, get_executable, get_platform, in_venv,
is_in_directory)
logger = logging.getLogger(__name__)
_DEFAULT_MANIFEST = '''
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
<assemblyIdentity version="1.0.0.0"
processorArchitecture="X86"
name="%s"
type="win32"/>
<!-- Identify the application security requirements. -->
<trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
<security>
<requestedPrivileges>
<requestedExecutionLevel level="asInvoker" uiAccess="false"/>
</requestedPrivileges>
</security>
</trustInfo>
</assembly>'''.strip()
# check if Python is called on the first line with this expression
FIRST_LINE_RE = re.compile(b'^#!.*pythonw?[0-9.]*([ \t].*)?$')
SCRIPT_TEMPLATE = r'''# -*- coding: utf-8 -*-
import re
import sys
if __name__ == '__main__':
from %(module)s import %(import_name)s
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(%(func)s())
'''
# Pre-fetch the contents of all executable wrapper stubs.
# This is to address https://github.com/pypa/pip/issues/12666.
# When updating pip, we rename the old pip in place before installing the
# new version. If we try to fetch a wrapper *after* that rename, the finder
# machinery will be confused as the package is no longer available at the
# location where it was imported from. So we load everything into memory in
# advance.
if os.name == 'nt' or (os.name == 'java' and os._name == 'nt'):
# Issue 31: don't hardcode an absolute package name, but
# determine it relative to the current package
DISTLIB_PACKAGE = __name__.rsplit('.', 1)[0]
WRAPPERS = {
r.name: r.bytes
for r in finder(DISTLIB_PACKAGE).iterator("")
if r.name.endswith(".exe")
}
def enquote_executable(executable):
if ' ' in executable:
# make sure we quote only the executable in case of env
# for example /usr/bin/env "/dir with spaces/bin/jython"
# instead of "/usr/bin/env /dir with spaces/bin/jython"
# otherwise whole
if executable.startswith('/usr/bin/env '):
env, _executable = executable.split(' ', 1)
if ' ' in _executable and not _executable.startswith('"'):
executable = '%s "%s"' % (env, _executable)
else:
if not executable.startswith('"'):
executable = '"%s"' % executable
return executable
# Keep the old name around (for now), as there is at least one project using it!
_enquote_executable = enquote_executable
class ScriptMaker(object):
"""
A class to copy or create scripts from source scripts or callable
specifications.
"""
script_template = SCRIPT_TEMPLATE
executable = None # for shebangs
def __init__(self, source_dir, target_dir, add_launchers=True, dry_run=False, fileop=None):
self.source_dir = source_dir
self.target_dir = target_dir
self.add_launchers = add_launchers
self.force = False
self.clobber = False
# It only makes sense to set mode bits on POSIX.
self.set_mode = (os.name == 'posix') or (os.name == 'java' and os._name == 'posix')
self.variants = set(('', 'X.Y'))
self._fileop = fileop or FileOperator(dry_run)
self._is_nt = os.name == 'nt' or (os.name == 'java' and os._name == 'nt')
self.version_info = sys.version_info
def _get_alternate_executable(self, executable, options):
if options.get('gui', False) and self._is_nt: # pragma: no cover
dn, fn = os.path.split(executable)
fn = fn.replace('python', 'pythonw')
executable = os.path.join(dn, fn)
return executable
if sys.platform.startswith('java'): # pragma: no cover
def _is_shell(self, executable):
"""
Determine if the specified executable is a script
(contains a #! line)
"""
try:
with open(executable) as fp:
return fp.read(2) == '#!'
except (OSError, IOError):
logger.warning('Failed to open %s', executable)
return False
def _fix_jython_executable(self, executable):
if self._is_shell(executable):
# Workaround for Jython is not needed on Linux systems.
import java
if java.lang.System.getProperty('os.name') == 'Linux':
return executable
elif executable.lower().endswith('jython.exe'):
# Use wrapper exe for Jython on Windows
return executable
return '/usr/bin/env %s' % executable
def _build_shebang(self, executable, post_interp):
"""
Build a shebang line. In the simple case (on Windows, or a shebang line
which is not too long or contains spaces) use a simple formulation for
the shebang. Otherwise, use /bin/sh as the executable, with a contrived
shebang which allows the script to run either under Python or sh, using
suitable quoting. Thanks to Harald Nordgren for his input.
See also: http://www.in-ulm.de/~mascheck/various/shebang/#length
https://hg.mozilla.org/mozilla-central/file/tip/mach
"""
if os.name != 'posix':
simple_shebang = True
elif getattr(sys, "cross_compiling", False):
# In a cross-compiling environment, the shebang will likely be a
# script; this *must* be invoked with the "safe" version of the
# shebang, or else using os.exec() to run the entry script will
# fail, raising "OSError 8 [Errno 8] Exec format error".
simple_shebang = False
else:
# Add 3 for '#!' prefix and newline suffix.
shebang_length = len(executable) + len(post_interp) + 3
if sys.platform == 'darwin':
max_shebang_length = 512
else:
max_shebang_length = 127
simple_shebang = ((b' ' not in executable) and (shebang_length <= max_shebang_length))
if simple_shebang:
result = b'#!' + executable + post_interp + b'\n'
else:
result = b'#!/bin/sh\n'
result += b"'''exec' " + executable + post_interp + b' "$0" "$@"\n'
result += b"' '''\n"
return result
def _get_shebang(self, encoding, post_interp=b'', options=None):
enquote = True
if self.executable:
executable = self.executable
enquote = False # assume this will be taken care of
elif not sysconfig.is_python_build():
executable = get_executable()
elif in_venv(): # pragma: no cover
executable = os.path.join(sysconfig.get_path('scripts'), 'python%s' % sysconfig.get_config_var('EXE'))
else: # pragma: no cover
if os.name == 'nt':
# for Python builds from source on Windows, no Python executables with
# a version suffix are created, so we use python.exe
executable = os.path.join(sysconfig.get_config_var('BINDIR'),
'python%s' % (sysconfig.get_config_var('EXE')))
else:
executable = os.path.join(
sysconfig.get_config_var('BINDIR'),
'python%s%s' % (sysconfig.get_config_var('VERSION'), sysconfig.get_config_var('EXE')))
if options:
executable = self._get_alternate_executable(executable, options)
if sys.platform.startswith('java'): # pragma: no cover
executable = self._fix_jython_executable(executable)
# Normalise case for Windows - COMMENTED OUT
# executable = os.path.normcase(executable)
# N.B. The normalising operation above has been commented out: See
# issue #124. Although paths in Windows are generally case-insensitive,
# they aren't always. For example, a path containing a ẞ (which is a
# LATIN CAPITAL LETTER SHARP S - U+1E9E) is normcased to ß (which is a
# LATIN SMALL LETTER SHARP S' - U+00DF). The two are not considered by
# Windows as equivalent in path names.
# If the user didn't specify an executable, it may be necessary to
# cater for executable paths with spaces (not uncommon on Windows)
if enquote:
executable = enquote_executable(executable)
# Issue #51: don't use fsencode, since we later try to
# check that the shebang is decodable using utf-8.
executable = executable.encode('utf-8')
# in case of IronPython, play safe and enable frames support
if (sys.platform == 'cli' and '-X:Frames' not in post_interp and
'-X:FullFrames' not in post_interp): # pragma: no cover
post_interp += b' -X:Frames'
shebang = self._build_shebang(executable, post_interp)
# Python parser starts to read a script using UTF-8 until
# it gets a #coding:xxx cookie. The shebang has to be the
# first line of a file, the #coding:xxx cookie cannot be
# written before. So the shebang has to be decodable from
# UTF-8.
try:
shebang.decode('utf-8')
except UnicodeDecodeError: # pragma: no cover
raise ValueError('The shebang (%r) is not decodable from utf-8' % shebang)
# If the script is encoded to a custom encoding (use a
# #coding:xxx cookie), the shebang has to be decodable from
# the script encoding too.
if encoding != 'utf-8':
try:
shebang.decode(encoding)
except UnicodeDecodeError: # pragma: no cover
raise ValueError('The shebang (%r) is not decodable '
'from the script encoding (%r)' % (shebang, encoding))
return shebang
def _get_script_text(self, entry):
return self.script_template % dict(
module=entry.prefix, import_name=entry.suffix.split('.')[0], func=entry.suffix)
manifest = _DEFAULT_MANIFEST
def get_manifest(self, exename):
base = os.path.basename(exename)
return self.manifest % base
def _write_script(self, names, shebang, script_bytes, filenames, ext):
use_launcher = self.add_launchers and self._is_nt
if not use_launcher:
script_bytes = shebang + script_bytes
else: # pragma: no cover
if ext == 'py':
launcher = self._get_launcher('t')
else:
launcher = self._get_launcher('w')
stream = BytesIO()
with ZipFile(stream, 'w') as zf:
source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH')
if source_date_epoch:
date_time = time.gmtime(int(source_date_epoch))[:6]
zinfo = ZipInfo(filename='__main__.py', date_time=date_time)
zf.writestr(zinfo, script_bytes)
else:
zf.writestr('__main__.py', script_bytes)
zip_data = stream.getvalue()
script_bytes = launcher + shebang + zip_data
for name in names:
outname = os.path.abspath(os.path.join(self.target_dir, name))
if not is_in_directory(outname, self.target_dir):
raise DistlibException('Attempt to escape script directory')
if use_launcher: # pragma: no cover
n, e = os.path.splitext(outname)
if e.startswith('.py'):
outname = n
outname = '%s.exe' % outname
try:
self._fileop.write_binary_file(outname, script_bytes)
except Exception:
# Failed writing an executable - it might be in use.
logger.warning('Failed to write executable - trying to '
'use .deleteme logic')
dfname = '%s.deleteme' % outname
if os.path.exists(dfname):
os.remove(dfname) # Not allowed to fail here
os.rename(outname, dfname) # nor here
self._fileop.write_binary_file(outname, script_bytes)
logger.debug('Able to replace executable using '
'.deleteme logic')
try:
os.remove(dfname)
except Exception:
pass # still in use - ignore error
else:
if self._is_nt and not outname.endswith('.' + ext): # pragma: no cover
outname = '%s.%s' % (outname, ext)
if os.path.exists(outname) and not self.clobber:
logger.warning('Skipping existing file %s', outname)
continue
self._fileop.write_binary_file(outname, script_bytes)
if self.set_mode:
self._fileop.set_executable_mode([outname])
filenames.append(outname)
variant_separator = '-'
def get_script_filenames(self, name):
result = set()
if '' in self.variants:
result.add(name)
if 'X' in self.variants:
result.add('%s%s' % (name, self.version_info[0]))
if 'X.Y' in self.variants:
result.add('%s%s%s.%s' % (name, self.variant_separator, self.version_info[0], self.version_info[1]))
return result
def _make_script(self, entry, filenames, options=None):
post_interp = b''
if options:
args = options.get('interpreter_args', [])
if args:
args = ' %s' % ' '.join(args)
post_interp = args.encode('utf-8')
shebang = self._get_shebang('utf-8', post_interp, options=options)
script = self._get_script_text(entry).encode('utf-8')
scriptnames = self.get_script_filenames(entry.name)
if options and options.get('gui', False):
ext = 'pyw'
else:
ext = 'py'
self._write_script(scriptnames, shebang, script, filenames, ext)
def _copy_script(self, script, filenames):
adjust = False
script = os.path.join(self.source_dir, convert_path(script))
outname = os.path.join(self.target_dir, os.path.basename(script))
if not self.force and not self._fileop.newer(script, outname):
logger.debug('not copying %s (up-to-date)', script)
return
# Always open the file, but ignore failures in dry-run mode --
# that way, we'll get accurate feedback if we can read the
# script.
try:
f = open(script, 'rb')
except IOError: # pragma: no cover
if not self.dry_run:
raise
f = None
else:
first_line = f.readline()
if not first_line: # pragma: no cover
logger.warning('%s is an empty file (skipping)', script)
return
match = FIRST_LINE_RE.match(first_line.replace(b'\r\n', b'\n'))
if match:
adjust = True
post_interp = match.group(1) or b''
if not adjust:
if f:
f.close()
self._fileop.copy_file(script, outname)
if self.set_mode:
self._fileop.set_executable_mode([outname])
filenames.append(outname)
else:
logger.info('copying and adjusting %s -> %s', script, self.target_dir)
if not self._fileop.dry_run:
encoding, lines = detect_encoding(f.readline)
f.seek(0)
shebang = self._get_shebang(encoding, post_interp)
if b'pythonw' in first_line: # pragma: no cover
ext = 'pyw'
else:
ext = 'py'
n = os.path.basename(outname)
self._write_script([n], shebang, f.read(), filenames, ext)
if f:
f.close()
@property
def dry_run(self):
return self._fileop.dry_run
@dry_run.setter
def dry_run(self, value):
self._fileop.dry_run = value
if os.name == 'nt' or (os.name == 'java' and os._name == 'nt'): # pragma: no cover
# Executable launcher support.
# Launchers are from https://bitbucket.org/vinay.sajip/simple_launcher/
def _get_launcher(self, kind):
if struct.calcsize('P') == 8: # 64-bit
bits = '64'
else:
bits = '32'
platform_suffix = '-arm' if get_platform() == 'win-arm64' else ''
name = '%s%s%s.exe' % (kind, bits, platform_suffix)
if name not in WRAPPERS:
msg = ('Unable to find resource %s in package %s' %
(name, DISTLIB_PACKAGE))
raise ValueError(msg)
return WRAPPERS[name]
# Public API follows
def make(self, specification, options=None):
"""
Make a script.
:param specification: The specification, which is either a valid export
entry specification (to make a script from a
callable) or a filename (to make a script by
copying from a source location).
:param options: A dictionary of options controlling script generation.
:return: A list of all absolute pathnames written to.
"""
filenames = []
entry = get_export_entry(specification)
if entry is None:
self._copy_script(specification, filenames)
else:
self._make_script(entry, filenames, options=options)
return filenames
def make_multiple(self, specifications, options=None):
"""
Take a list of specifications and make scripts from them,
:param specifications: A list of specifications.
:return: A list of all absolute pathnames written to,
"""
filenames = []
for specification in specifications:
filenames.extend(self.make(specification, options))
return filenames
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,750 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2012-2023 The Python Software Foundation.
# See LICENSE.txt and CONTRIBUTORS.txt.
#
"""
Implementation of a flexible versioning scheme providing support for PEP-440,
setuptools-compatible and semantic versioning.
"""
import logging
import re
from .compat import string_types
from .util import parse_requirement
__all__ = ['NormalizedVersion', 'NormalizedMatcher',
'LegacyVersion', 'LegacyMatcher',
'SemanticVersion', 'SemanticMatcher',
'UnsupportedVersionError', 'get_scheme']
logger = logging.getLogger(__name__)
class UnsupportedVersionError(ValueError):
"""This is an unsupported version."""
pass
class Version(object):
def __init__(self, s):
self._string = s = s.strip()
self._parts = parts = self.parse(s)
assert isinstance(parts, tuple)
assert len(parts) > 0
def parse(self, s):
raise NotImplementedError('please implement in a subclass')
def _check_compatible(self, other):
if type(self) != type(other):
raise TypeError('cannot compare %r and %r' % (self, other))
def __eq__(self, other):
self._check_compatible(other)
return self._parts == other._parts
def __ne__(self, other):
return not self.__eq__(other)
def __lt__(self, other):
self._check_compatible(other)
return self._parts < other._parts
def __gt__(self, other):
return not (self.__lt__(other) or self.__eq__(other))
def __le__(self, other):
return self.__lt__(other) or self.__eq__(other)
def __ge__(self, other):
return self.__gt__(other) or self.__eq__(other)
# See http://docs.python.org/reference/datamodel#object.__hash__
def __hash__(self):
return hash(self._parts)
def __repr__(self):
return "%s('%s')" % (self.__class__.__name__, self._string)
def __str__(self):
return self._string
@property
def is_prerelease(self):
raise NotImplementedError('Please implement in subclasses.')
class Matcher(object):
version_class = None
# value is either a callable or the name of a method
_operators = {
'<': lambda v, c, p: v < c,
'>': lambda v, c, p: v > c,
'<=': lambda v, c, p: v == c or v < c,
'>=': lambda v, c, p: v == c or v > c,
'==': lambda v, c, p: v == c,
'===': lambda v, c, p: v == c,
# by default, compatible => >=.
'~=': lambda v, c, p: v == c or v > c,
'!=': lambda v, c, p: v != c,
}
# this is a method only to support alternative implementations
# via overriding
def parse_requirement(self, s):
return parse_requirement(s)
def __init__(self, s):
if self.version_class is None:
raise ValueError('Please specify a version class')
self._string = s = s.strip()
r = self.parse_requirement(s)
if not r:
raise ValueError('Not valid: %r' % s)
self.name = r.name
self.key = self.name.lower() # for case-insensitive comparisons
clist = []
if r.constraints:
# import pdb; pdb.set_trace()
for op, s in r.constraints:
if s.endswith('.*'):
if op not in ('==', '!='):
raise ValueError('\'.*\' not allowed for '
'%r constraints' % op)
# Could be a partial version (e.g. for '2.*') which
# won't parse as a version, so keep it as a string
vn, prefix = s[:-2], True
# Just to check that vn is a valid version
self.version_class(vn)
else:
# Should parse as a version, so we can create an
# instance for the comparison
vn, prefix = self.version_class(s), False
clist.append((op, vn, prefix))
self._parts = tuple(clist)
def match(self, version):
"""
Check if the provided version matches the constraints.
:param version: The version to match against this instance.
:type version: String or :class:`Version` instance.
"""
if isinstance(version, string_types):
version = self.version_class(version)
for operator, constraint, prefix in self._parts:
f = self._operators.get(operator)
if isinstance(f, string_types):
f = getattr(self, f)
if not f:
msg = ('%r not implemented '
'for %s' % (operator, self.__class__.__name__))
raise NotImplementedError(msg)
if not f(version, constraint, prefix):
return False
return True
@property
def exact_version(self):
result = None
if len(self._parts) == 1 and self._parts[0][0] in ('==', '==='):
result = self._parts[0][1]
return result
def _check_compatible(self, other):
if type(self) != type(other) or self.name != other.name:
raise TypeError('cannot compare %s and %s' % (self, other))
def __eq__(self, other):
self._check_compatible(other)
return self.key == other.key and self._parts == other._parts
def __ne__(self, other):
return not self.__eq__(other)
# See http://docs.python.org/reference/datamodel#object.__hash__
def __hash__(self):
return hash(self.key) + hash(self._parts)
def __repr__(self):
return "%s(%r)" % (self.__class__.__name__, self._string)
def __str__(self):
return self._string
PEP440_VERSION_RE = re.compile(r'^v?(\d+!)?(\d+(\.\d+)*)((a|alpha|b|beta|c|rc|pre|preview)(\d+)?)?'
r'(\.(post|r|rev)(\d+)?)?([._-]?(dev)(\d+)?)?'
r'(\+([a-zA-Z\d]+(\.[a-zA-Z\d]+)?))?$', re.I)
def _pep_440_key(s):
s = s.strip()
m = PEP440_VERSION_RE.match(s)
if not m:
raise UnsupportedVersionError('Not a valid version: %s' % s)
groups = m.groups()
nums = tuple(int(v) for v in groups[1].split('.'))
while len(nums) > 1 and nums[-1] == 0:
nums = nums[:-1]
if not groups[0]:
epoch = 0
else:
epoch = int(groups[0][:-1])
pre = groups[4:6]
post = groups[7:9]
dev = groups[10:12]
local = groups[13]
if pre == (None, None):
pre = ()
else:
if pre[1] is None:
pre = pre[0], 0
else:
pre = pre[0], int(pre[1])
if post == (None, None):
post = ()
else:
if post[1] is None:
post = post[0], 0
else:
post = post[0], int(post[1])
if dev == (None, None):
dev = ()
else:
if dev[1] is None:
dev = dev[0], 0
else:
dev = dev[0], int(dev[1])
if local is None:
local = ()
else:
parts = []
for part in local.split('.'):
# to ensure that numeric compares as > lexicographic, avoid
# comparing them directly, but encode a tuple which ensures
# correct sorting
if part.isdigit():
part = (1, int(part))
else:
part = (0, part)
parts.append(part)
local = tuple(parts)
if not pre:
# either before pre-release, or final release and after
if not post and dev:
# before pre-release
pre = ('a', -1) # to sort before a0
else:
pre = ('z',) # to sort after all pre-releases
# now look at the state of post and dev.
if not post:
post = ('_',) # sort before 'a'
if not dev:
dev = ('final',)
return epoch, nums, pre, post, dev, local
_normalized_key = _pep_440_key
class NormalizedVersion(Version):
"""A rational version.
Good:
1.2 # equivalent to "1.2.0"
1.2.0
1.2a1
1.2.3a2
1.2.3b1
1.2.3c1
1.2.3.4
TODO: fill this out
Bad:
1 # minimum two numbers
1.2a # release level must have a release serial
1.2.3b
"""
def parse(self, s):
result = _normalized_key(s)
# _normalized_key loses trailing zeroes in the release
# clause, since that's needed to ensure that X.Y == X.Y.0 == X.Y.0.0
# However, PEP 440 prefix matching needs it: for example,
# (~= 1.4.5.0) matches differently to (~= 1.4.5.0.0).
m = PEP440_VERSION_RE.match(s) # must succeed
groups = m.groups()
self._release_clause = tuple(int(v) for v in groups[1].split('.'))
return result
PREREL_TAGS = set(['a', 'b', 'c', 'rc', 'dev'])
@property
def is_prerelease(self):
return any(t[0] in self.PREREL_TAGS for t in self._parts if t)
def _match_prefix(x, y):
x = str(x)
y = str(y)
if x == y:
return True
if not x.startswith(y):
return False
n = len(y)
return x[n] == '.'
class NormalizedMatcher(Matcher):
version_class = NormalizedVersion
# value is either a callable or the name of a method
_operators = {
'~=': '_match_compatible',
'<': '_match_lt',
'>': '_match_gt',
'<=': '_match_le',
'>=': '_match_ge',
'==': '_match_eq',
'===': '_match_arbitrary',
'!=': '_match_ne',
}
def _adjust_local(self, version, constraint, prefix):
if prefix:
strip_local = '+' not in constraint and version._parts[-1]
else:
# both constraint and version are
# NormalizedVersion instances.
# If constraint does not have a local component,
# ensure the version doesn't, either.
strip_local = not constraint._parts[-1] and version._parts[-1]
if strip_local:
s = version._string.split('+', 1)[0]
version = self.version_class(s)
return version, constraint
def _match_lt(self, version, constraint, prefix):
version, constraint = self._adjust_local(version, constraint, prefix)
if version >= constraint:
return False
release_clause = constraint._release_clause
pfx = '.'.join([str(i) for i in release_clause])
return not _match_prefix(version, pfx)
def _match_gt(self, version, constraint, prefix):
version, constraint = self._adjust_local(version, constraint, prefix)
if version <= constraint:
return False
release_clause = constraint._release_clause
pfx = '.'.join([str(i) for i in release_clause])
return not _match_prefix(version, pfx)
def _match_le(self, version, constraint, prefix):
version, constraint = self._adjust_local(version, constraint, prefix)
return version <= constraint
def _match_ge(self, version, constraint, prefix):
version, constraint = self._adjust_local(version, constraint, prefix)
return version >= constraint
def _match_eq(self, version, constraint, prefix):
version, constraint = self._adjust_local(version, constraint, prefix)
if not prefix:
result = (version == constraint)
else:
result = _match_prefix(version, constraint)
return result
def _match_arbitrary(self, version, constraint, prefix):
return str(version) == str(constraint)
def _match_ne(self, version, constraint, prefix):
version, constraint = self._adjust_local(version, constraint, prefix)
if not prefix:
result = (version != constraint)
else:
result = not _match_prefix(version, constraint)
return result
def _match_compatible(self, version, constraint, prefix):
version, constraint = self._adjust_local(version, constraint, prefix)
if version == constraint:
return True
if version < constraint:
return False
# if not prefix:
# return True
release_clause = constraint._release_clause
if len(release_clause) > 1:
release_clause = release_clause[:-1]
pfx = '.'.join([str(i) for i in release_clause])
return _match_prefix(version, pfx)
_REPLACEMENTS = (
(re.compile('[.+-]$'), ''), # remove trailing puncts
(re.compile(r'^[.](\d)'), r'0.\1'), # .N -> 0.N at start
(re.compile('^[.-]'), ''), # remove leading puncts
(re.compile(r'^\((.*)\)$'), r'\1'), # remove parentheses
(re.compile(r'^v(ersion)?\s*(\d+)'), r'\2'), # remove leading v(ersion)
(re.compile(r'^r(ev)?\s*(\d+)'), r'\2'), # remove leading v(ersion)
(re.compile('[.]{2,}'), '.'), # multiple runs of '.'
(re.compile(r'\b(alfa|apha)\b'), 'alpha'), # misspelt alpha
(re.compile(r'\b(pre-alpha|prealpha)\b'),
'pre.alpha'), # standardise
(re.compile(r'\(beta\)$'), 'beta'), # remove parentheses
)
_SUFFIX_REPLACEMENTS = (
(re.compile('^[:~._+-]+'), ''), # remove leading puncts
(re.compile('[,*")([\\]]'), ''), # remove unwanted chars
(re.compile('[~:+_ -]'), '.'), # replace illegal chars
(re.compile('[.]{2,}'), '.'), # multiple runs of '.'
(re.compile(r'\.$'), ''), # trailing '.'
)
_NUMERIC_PREFIX = re.compile(r'(\d+(\.\d+)*)')
def _suggest_semantic_version(s):
"""
Try to suggest a semantic form for a version for which
_suggest_normalized_version couldn't come up with anything.
"""
result = s.strip().lower()
for pat, repl in _REPLACEMENTS:
result = pat.sub(repl, result)
if not result:
result = '0.0.0'
# Now look for numeric prefix, and separate it out from
# the rest.
# import pdb; pdb.set_trace()
m = _NUMERIC_PREFIX.match(result)
if not m:
prefix = '0.0.0'
suffix = result
else:
prefix = m.groups()[0].split('.')
prefix = [int(i) for i in prefix]
while len(prefix) < 3:
prefix.append(0)
if len(prefix) == 3:
suffix = result[m.end():]
else:
suffix = '.'.join([str(i) for i in prefix[3:]]) + result[m.end():]
prefix = prefix[:3]
prefix = '.'.join([str(i) for i in prefix])
suffix = suffix.strip()
if suffix:
# import pdb; pdb.set_trace()
# massage the suffix.
for pat, repl in _SUFFIX_REPLACEMENTS:
suffix = pat.sub(repl, suffix)
if not suffix:
result = prefix
else:
sep = '-' if 'dev' in suffix else '+'
result = prefix + sep + suffix
if not is_semver(result):
result = None
return result
def _suggest_normalized_version(s):
"""Suggest a normalized version close to the given version string.
If you have a version string that isn't rational (i.e. NormalizedVersion
doesn't like it) then you might be able to get an equivalent (or close)
rational version from this function.
This does a number of simple normalizations to the given string, based
on observation of versions currently in use on PyPI. Given a dump of
those version during PyCon 2009, 4287 of them:
- 2312 (53.93%) match NormalizedVersion without change
with the automatic suggestion
- 3474 (81.04%) match when using this suggestion method
@param s {str} An irrational version string.
@returns A rational version string, or None, if couldn't determine one.
"""
try:
_normalized_key(s)
return s # already rational
except UnsupportedVersionError:
pass
rs = s.lower()
# part of this could use maketrans
for orig, repl in (('-alpha', 'a'), ('-beta', 'b'), ('alpha', 'a'),
('beta', 'b'), ('rc', 'c'), ('-final', ''),
('-pre', 'c'),
('-release', ''), ('.release', ''), ('-stable', ''),
('+', '.'), ('_', '.'), (' ', ''), ('.final', ''),
('final', '')):
rs = rs.replace(orig, repl)
# if something ends with dev or pre, we add a 0
rs = re.sub(r"pre$", r"pre0", rs)
rs = re.sub(r"dev$", r"dev0", rs)
# if we have something like "b-2" or "a.2" at the end of the
# version, that is probably beta, alpha, etc
# let's remove the dash or dot
rs = re.sub(r"([abc]|rc)[\-\.](\d+)$", r"\1\2", rs)
# 1.0-dev-r371 -> 1.0.dev371
# 0.1-dev-r79 -> 0.1.dev79
rs = re.sub(r"[\-\.](dev)[\-\.]?r?(\d+)$", r".\1\2", rs)
# Clean: 2.0.a.3, 2.0.b1, 0.9.0~c1
rs = re.sub(r"[.~]?([abc])\.?", r"\1", rs)
# Clean: v0.3, v1.0
if rs.startswith('v'):
rs = rs[1:]
# Clean leading '0's on numbers.
# TODO: unintended side-effect on, e.g., "2003.05.09"
# PyPI stats: 77 (~2%) better
rs = re.sub(r"\b0+(\d+)(?!\d)", r"\1", rs)
# Clean a/b/c with no version. E.g. "1.0a" -> "1.0a0". Setuptools infers
# zero.
# PyPI stats: 245 (7.56%) better
rs = re.sub(r"(\d+[abc])$", r"\g<1>0", rs)
# the 'dev-rNNN' tag is a dev tag
rs = re.sub(r"\.?(dev-r|dev\.r)\.?(\d+)$", r".dev\2", rs)
# clean the - when used as a pre delimiter
rs = re.sub(r"-(a|b|c)(\d+)$", r"\1\2", rs)
# a terminal "dev" or "devel" can be changed into ".dev0"
rs = re.sub(r"[\.\-](dev|devel)$", r".dev0", rs)
# a terminal "dev" can be changed into ".dev0"
rs = re.sub(r"(?![\.\-])dev$", r".dev0", rs)
# a terminal "final" or "stable" can be removed
rs = re.sub(r"(final|stable)$", "", rs)
# The 'r' and the '-' tags are post release tags
# 0.4a1.r10 -> 0.4a1.post10
# 0.9.33-17222 -> 0.9.33.post17222
# 0.9.33-r17222 -> 0.9.33.post17222
rs = re.sub(r"\.?(r|-|-r)\.?(\d+)$", r".post\2", rs)
# Clean 'r' instead of 'dev' usage:
# 0.9.33+r17222 -> 0.9.33.dev17222
# 1.0dev123 -> 1.0.dev123
# 1.0.git123 -> 1.0.dev123
# 1.0.bzr123 -> 1.0.dev123
# 0.1a0dev.123 -> 0.1a0.dev123
# PyPI stats: ~150 (~4%) better
rs = re.sub(r"\.?(dev|git|bzr)\.?(\d+)$", r".dev\2", rs)
# Clean '.pre' (normalized from '-pre' above) instead of 'c' usage:
# 0.2.pre1 -> 0.2c1
# 0.2-c1 -> 0.2c1
# 1.0preview123 -> 1.0c123
# PyPI stats: ~21 (0.62%) better
rs = re.sub(r"\.?(pre|preview|-c)(\d+)$", r"c\g<2>", rs)
# Tcl/Tk uses "px" for their post release markers
rs = re.sub(r"p(\d+)$", r".post\1", rs)
try:
_normalized_key(rs)
except UnsupportedVersionError:
rs = None
return rs
#
# Legacy version processing (distribute-compatible)
#
_VERSION_PART = re.compile(r'([a-z]+|\d+|[\.-])', re.I)
_VERSION_REPLACE = {
'pre': 'c',
'preview': 'c',
'-': 'final-',
'rc': 'c',
'dev': '@',
'': None,
'.': None,
}
def _legacy_key(s):
def get_parts(s):
result = []
for p in _VERSION_PART.split(s.lower()):
p = _VERSION_REPLACE.get(p, p)
if p:
if '0' <= p[:1] <= '9':
p = p.zfill(8)
else:
p = '*' + p
result.append(p)
result.append('*final')
return result
result = []
for p in get_parts(s):
if p.startswith('*'):
if p < '*final':
while result and result[-1] == '*final-':
result.pop()
while result and result[-1] == '00000000':
result.pop()
result.append(p)
return tuple(result)
class LegacyVersion(Version):
def parse(self, s):
return _legacy_key(s)
@property
def is_prerelease(self):
result = False
for x in self._parts:
if (isinstance(x, string_types) and x.startswith('*') and x < '*final'):
result = True
break
return result
class LegacyMatcher(Matcher):
version_class = LegacyVersion
_operators = dict(Matcher._operators)
_operators['~='] = '_match_compatible'
numeric_re = re.compile(r'^(\d+(\.\d+)*)')
def _match_compatible(self, version, constraint, prefix):
if version < constraint:
return False
m = self.numeric_re.match(str(constraint))
if not m:
logger.warning('Cannot compute compatible match for version %s '
' and constraint %s', version, constraint)
return True
s = m.groups()[0]
if '.' in s:
s = s.rsplit('.', 1)[0]
return _match_prefix(version, s)
#
# Semantic versioning
#
_SEMVER_RE = re.compile(r'^(\d+)\.(\d+)\.(\d+)'
r'(-[a-z0-9]+(\.[a-z0-9-]+)*)?'
r'(\+[a-z0-9]+(\.[a-z0-9-]+)*)?$', re.I)
def is_semver(s):
return _SEMVER_RE.match(s)
def _semantic_key(s):
def make_tuple(s, absent):
if s is None:
result = (absent,)
else:
parts = s[1:].split('.')
# We can't compare ints and strings on Python 3, so fudge it
# by zero-filling numeric values so simulate a numeric comparison
result = tuple([p.zfill(8) if p.isdigit() else p for p in parts])
return result
m = is_semver(s)
if not m:
raise UnsupportedVersionError(s)
groups = m.groups()
major, minor, patch = [int(i) for i in groups[:3]]
# choose the '|' and '*' so that versions sort correctly
pre, build = make_tuple(groups[3], '|'), make_tuple(groups[5], '*')
return (major, minor, patch), pre, build
class SemanticVersion(Version):
def parse(self, s):
return _semantic_key(s)
@property
def is_prerelease(self):
return self._parts[1][0] != '|'
class SemanticMatcher(Matcher):
version_class = SemanticVersion
class VersionScheme(object):
def __init__(self, key, matcher, suggester=None):
self.key = key
self.matcher = matcher
self.suggester = suggester
def is_valid_version(self, s):
try:
self.matcher.version_class(s)
result = True
except UnsupportedVersionError:
result = False
return result
def is_valid_matcher(self, s):
try:
self.matcher(s)
result = True
except UnsupportedVersionError:
result = False
return result
def is_valid_constraint_list(self, s):
"""
Used for processing some metadata fields
"""
# See issue #140. Be tolerant of a single trailing comma.
if s.endswith(','):
s = s[:-1]
return self.is_valid_matcher('dummy_name (%s)' % s)
def suggest(self, s):
if self.suggester is None:
result = None
else:
result = self.suggester(s)
return result
_SCHEMES = {
'normalized': VersionScheme(_normalized_key, NormalizedMatcher,
_suggest_normalized_version),
'legacy': VersionScheme(_legacy_key, LegacyMatcher, lambda self, s: s),
'semantic': VersionScheme(_semantic_key, SemanticMatcher,
_suggest_semantic_version),
}
_SCHEMES['default'] = _SCHEMES['normalized']
def get_scheme(name):
if name not in _SCHEMES:
raise ValueError('unknown scheme name: %r' % name)
return _SCHEMES[name]
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large Load Diff
@@ -0,0 +1 @@
import os; var = 'SETUPTOOLS_USE_DISTUTILS'; enabled = os.environ.get(var, 'local') == 'local'; enabled and __import__('_distutils_hack').add_shim();
@@ -0,0 +1 @@
pip
@@ -0,0 +1,38 @@
Metadata-Version: 2.4
Name: filelock
Version: 3.29.7
Summary: A platform independent file lock.
Project-URL: Documentation, https://py-filelock.readthedocs.io
Project-URL: Homepage, https://github.com/tox-dev/py-filelock
Project-URL: Source, https://github.com/tox-dev/py-filelock
Project-URL: Tracker, https://github.com/tox-dev/py-filelock/issues
Maintainer-email: Bernát Gábor <gaborjbernat@gmail.com>
License-Expression: MIT
License-File: LICENSE
Keywords: application,cache,directory,log,user
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: 3.14
Classifier: Topic :: Internet
Classifier: Topic :: Software Development :: Libraries
Classifier: Topic :: System
Requires-Python: >=3.10
Description-Content-Type: text/markdown
# filelock
[![PyPI](https://img.shields.io/pypi/v/filelock)](https://pypi.org/project/filelock/)
[![Supported Python versions](https://img.shields.io/pypi/pyversions/filelock.svg)](https://pypi.org/project/filelock/)
[![Documentation status](https://readthedocs.org/projects/py-filelock/badge/?version=latest)](https://py-filelock.readthedocs.io/en/latest/?badge=latest)
[![Downloads](https://static.pepy.tech/badge/filelock/month)](https://pepy.tech/project/filelock)
[![check](https://github.com/tox-dev/py-filelock/actions/workflows/check.yaml/badge.svg)](https://github.com/tox-dev/py-filelock/actions/workflows/check.yaml)
For more information checkout the [official documentation](https://py-filelock.readthedocs.io/en/latest/index.html).
@@ -0,0 +1,34 @@
filelock-3.29.7.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
filelock-3.29.7.dist-info/METADATA,sha256=J1CqEhZ6lw6FJNU8s65mcMmuiJS-yaqMIWptsV_h0Sg,1977
filelock-3.29.7.dist-info/RECORD,,
filelock-3.29.7.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
filelock-3.29.7.dist-info/licenses/LICENSE,sha256=YIyJ1QYK6ZIa3M8yNmlbxlSplG4SMj72wCHfoE4pTUg,1088
filelock/__init__.py,sha256=Ig_QNvsAVtctz2-fkq9EUCgXmW6M5PDudiOC85szmis,2612
filelock/__pycache__/__init__.cpython-311.pyc,,
filelock/__pycache__/_api.cpython-311.pyc,,
filelock/__pycache__/_async_read_write.cpython-311.pyc,,
filelock/__pycache__/_error.cpython-311.pyc,,
filelock/__pycache__/_read_write.cpython-311.pyc,,
filelock/__pycache__/_soft.cpython-311.pyc,,
filelock/__pycache__/_unix.cpython-311.pyc,,
filelock/__pycache__/_util.cpython-311.pyc,,
filelock/__pycache__/_windows.cpython-311.pyc,,
filelock/__pycache__/asyncio.cpython-311.pyc,,
filelock/__pycache__/version.cpython-311.pyc,,
filelock/_api.py,sha256=m1MU_Tx_i6X0lFCrLKreTOIO7PLkyvQk1hWCmd_RyGk,25025
filelock/_async_read_write.py,sha256=cI59FlQ1BaebdmoG77OVcTKYaDVabPJOjh5xsOGYC8c,8457
filelock/_error.py,sha256=IytAfHRl_zagaLkbto73QkiWni0KfTzq-6Xgc1v6450,765
filelock/_read_write.py,sha256=f6oAIkzwX2cEI_8Y2haeo49yL79gZ1Y3vLpNRGTiHcg,17030
filelock/_soft.py,sha256=z7I5KJFNZewCZdVQS8Ploq80tKKurLtxCQxPptRbzQA,10329
filelock/_soft_rw/__init__.py,sha256=_ktpGmVObzrvL0K1_EAV3LF7B9JarYxW-mx_eyGPhT0,370
filelock/_soft_rw/__pycache__/__init__.cpython-311.pyc,,
filelock/_soft_rw/__pycache__/_async.cpython-311.pyc,,
filelock/_soft_rw/__pycache__/_sync.cpython-311.pyc,,
filelock/_soft_rw/_async.py,sha256=Hu7e8epFROEzkEGW1Qq-oe25O0fxl6lVilMM6ORkSFE,8654
filelock/_soft_rw/_sync.py,sha256=355gvnzs2PnwAHv50WhCAexUBxWeg3CT6QL0KTeme_M,40857
filelock/_unix.py,sha256=zNmZv3F3Qqkl_r9rxI2S5s5eKdh3y6G1rHFb0a5G-Zc,4505
filelock/_util.py,sha256=HDc7Kwnpy_dUNW4imdh-ewYn9BRixQASI6jwgVHUAew,4446
filelock/_windows.py,sha256=kJFOenaffOP6Qza2b3-yxgiPiDcyroWnzz9STGkCrgM,3943
filelock/asyncio.py,sha256=FDnTEgwvXf7sHy7vqnh4I5Isq3x4RT5SEBtzh8GxOuQ,15795
filelock/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
filelock/version.py,sha256=3Kg5hZP5lsDTggGSFyyGB5sau1LWFEM4TTzXS6J857I,522
@@ -0,0 +1,4 @@
Wheel-Version: 1.0
Generator: hatchling 1.31.0
Root-Is-Purelib: true
Tag: py3-none-any
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 Bernát Gábor and contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@@ -0,0 +1,93 @@
"""
A platform independent file lock that supports the with-statement.
.. autodata:: filelock.__version__
:no-value:
"""
from __future__ import annotations
import sys
import warnings
from typing import TYPE_CHECKING
from ._api import AcquireReturnProxy, BaseFileLock
from ._error import Timeout
if TYPE_CHECKING:
from ._async_read_write import (
AsyncAcquireReadWriteReturnProxy,
AsyncReadWriteLock,
)
from ._read_write import ReadWriteLock
else:
try:
from ._async_read_write import AsyncAcquireReadWriteReturnProxy, AsyncReadWriteLock
from ._read_write import ReadWriteLock
except ImportError: # sqlite3 may be unavailable if Python was built without it or the C library is missing
AsyncAcquireReadWriteReturnProxy = None
AsyncReadWriteLock = None
ReadWriteLock = None
from ._soft import SoftFileLock
from ._soft_rw import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock, SoftReadWriteLock
from ._unix import UnixFileLock, has_fcntl
from ._windows import WindowsFileLock
from .asyncio import (
AsyncAcquireReturnProxy,
AsyncSoftFileLock,
AsyncUnixFileLock,
AsyncWindowsFileLock,
BaseAsyncFileLock,
)
from .version import version
#: version of the project as a string
__version__: str = version
if sys.platform == "win32": # pragma: win32 cover
_FileLock: type[BaseFileLock] = WindowsFileLock
_AsyncFileLock: type[BaseAsyncFileLock] = AsyncWindowsFileLock
else: # pragma: win32 no cover # noqa: PLR5501
if has_fcntl:
_FileLock: type[BaseFileLock] = UnixFileLock
_AsyncFileLock: type[BaseAsyncFileLock] = AsyncUnixFileLock
else:
_FileLock = SoftFileLock
_AsyncFileLock = AsyncSoftFileLock
if warnings is not None:
warnings.warn("only soft file lock is available", stacklevel=2)
if TYPE_CHECKING:
FileLock = SoftFileLock
AsyncFileLock = AsyncSoftFileLock
else:
#: Alias for the lock, which should be used for the current platform.
FileLock = _FileLock
AsyncFileLock = _AsyncFileLock
__all__ = [
"AcquireReturnProxy",
"AsyncAcquireReadWriteReturnProxy",
"AsyncAcquireReturnProxy",
"AsyncAcquireSoftReadWriteReturnProxy",
"AsyncFileLock",
"AsyncReadWriteLock",
"AsyncSoftFileLock",
"AsyncSoftReadWriteLock",
"AsyncUnixFileLock",
"AsyncWindowsFileLock",
"BaseAsyncFileLock",
"BaseFileLock",
"FileLock",
"ReadWriteLock",
"SoftFileLock",
"SoftReadWriteLock",
"Timeout",
"UnixFileLock",
"WindowsFileLock",
"__version__",
]
+645
View File
@@ -0,0 +1,645 @@
from __future__ import annotations
import contextlib
import inspect
import logging
import os
import sys
import time
import warnings
from abc import ABCMeta, abstractmethod
from dataclasses import dataclass
from threading import Lock, local
from typing import TYPE_CHECKING, Any, TypeVar
from weakref import WeakValueDictionary
from ._error import Timeout
from ._util import break_lock_file
#: Sentinel indicating that no explicit file permission mode was passed.
#: When used, lock files are created with 0o666 (letting umask and default ACLs control the final permissions)
#: and fchmod is skipped so that POSIX default ACL inheritance is preserved.
_UNSET_FILE_MODE: int = -1
if TYPE_CHECKING:
from collections.abc import Callable
from types import TracebackType
from ._read_write import ReadWriteLock
from ._soft_rw import SoftReadWriteLock
if sys.version_info >= (3, 11): # pragma: no cover (py311+)
from typing import Self
else: # pragma: no cover (<py311)
from typing_extensions import Self
_LOGGER = logging.getLogger("filelock")
# On Windows os.path.realpath calls CreateFileW with share_mode=0, which blocks concurrent DeleteFileW and causes
# livelocks under threaded contention with SoftFileLock. os.path.abspath is purely string-based and avoids this.
_canonical = os.path.abspath if sys.platform == "win32" else os.path.realpath
class _ThreadLocalRegistry(local):
def __init__(self) -> None:
super().__init__()
self.held: dict[str, int] = {}
_registry = _ThreadLocalRegistry()
# This is a helper class which is returned by :meth:`BaseFileLock.acquire` and wraps the lock to make sure __enter__
# is not called twice when entering the with statement. If we would simply return *self*, the lock would be acquired
# again in the *__enter__* method of the BaseFileLock, but not released again automatically. issue #37 (memory leak)
class AcquireReturnProxy:
"""A context-aware object that will release the lock file when exiting."""
def __init__(self, lock: BaseFileLock | ReadWriteLock | SoftReadWriteLock) -> None:
self.lock: BaseFileLock | ReadWriteLock | SoftReadWriteLock = lock
def __enter__(self) -> BaseFileLock | ReadWriteLock | SoftReadWriteLock:
return self.lock
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
self.lock.release()
@dataclass
class FileLockContext:
"""A dataclass which holds the context for a ``BaseFileLock`` object."""
# The context is held in a separate class to allow optional use of thread local storage via the
# ThreadLocalFileContext class.
#: The path to the lock file.
lock_file: str
#: The default timeout value.
timeout: float
#: The mode for the lock files
mode: int
#: Whether the lock should be blocking or not
blocking: bool
#: The default polling interval value.
poll_interval: float
#: The lock lifetime in seconds; ``None`` means the lock never expires.
lifetime: float | None = None
#: The file descriptor for the *_lock_file* as it is returned by the os.open() function, not None when lock held
lock_file_fd: int | None = None
#: The lock counter is used for implementing the nested locking mechanism.
lock_counter: int = 0 # When the lock is acquired is increased and the lock is only released, when this value is 0
class ThreadLocalFileContext(FileLockContext, local):
"""A thread local version of the ``FileLockContext`` class."""
_T = TypeVar("_T", bound="BaseFileLock")
class FileLockMeta(ABCMeta):
_instances: WeakValueDictionary[str, BaseFileLock]
_instances_lock: Lock
def __call__( # noqa: PLR0913
cls: type[_T],
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = True, # noqa: FBT001, FBT002
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
**kwargs: Any, # capture remaining kwargs for subclasses # noqa: ANN401
) -> _T:
params = {
"timeout": timeout,
"mode": mode,
"thread_local": thread_local,
"blocking": blocking,
"is_singleton": is_singleton,
"poll_interval": poll_interval,
"lifetime": lifetime,
**kwargs,
}
if not is_singleton:
return cls._create_instance(lock_file, params)
# Look up, build and store under one lock. Without it two threads racing the first construction for a
# path both miss the cache and each build their own instance, so callers relying on is_singleton for
# reentrant locking across instances end up with two "singletons" and acquire()'s deadlock check then
# rejects a legitimate reentrant acquire; the unguarded writes to the WeakValueDictionary are a data
# race besides. ReadWriteLock and SoftReadWriteLock already guard their singleton caches this way.
with cls._instances_lock:
if (instance := cls._instances.get(str(lock_file))) is None:
instance = cls._create_instance(lock_file, params)
cls._instances[str(lock_file)] = instance
return instance
params_to_check = {
"thread_local": (thread_local, instance.is_thread_local()),
"timeout": (timeout, instance.timeout),
"mode": (mode, instance._context.mode), # noqa: SLF001
"blocking": (blocking, instance.blocking),
"poll_interval": (poll_interval, instance.poll_interval),
"lifetime": (lifetime, instance.lifetime),
}
non_matching_params = {
name: (passed_param, set_param)
for name, (passed_param, set_param) in params_to_check.items()
if passed_param != set_param
}
if not non_matching_params:
return instance # ty: ignore[invalid-return-type] # https://github.com/astral-sh/ty/issues/3231
# parameters do not match; raise error
msg = "Singleton lock instances cannot be initialized with differing arguments"
msg += "\nNon-matching arguments: "
for param_name, (passed_param, set_param) in non_matching_params.items():
msg += f"\n\t{param_name} (existing lock has {set_param} but {passed_param} was passed)"
raise ValueError(msg)
def _create_instance(cls: type[_T], lock_file: str | os.PathLike[str], params: dict[str, Any]) -> _T:
# Keep only the params this subclass's __init__ accepts: virtualenv narrows the signature of its
# BaseFileLock descendant, so passing the full set would break it (https://github.com/tox-dev/filelock/pull/340).
present_params = inspect.signature(cls.__init__).parameters
init_params = {key: value for key, value in params.items() if key in present_params}
return super().__call__(lock_file, **init_params)
class BaseFileLock(contextlib.ContextDecorator, metaclass=FileLockMeta):
"""
Abstract base class for a file lock object.
Provides a reentrant, cross-process exclusive lock backed by OS-level primitives. Subclasses implement the actual
locking mechanism (:class:`UnixFileLock <filelock.UnixFileLock>`, :class:`WindowsFileLock
<filelock.WindowsFileLock>`, :class:`SoftFileLock <filelock.SoftFileLock>`).
"""
_instances: WeakValueDictionary[str, BaseFileLock]
_instances_lock: Lock
#: How the cross-instance deadlock message names the conflicting holder; the async subclass says "task".
_deadlock_holder_desc: str = "FileLock instance in this thread"
def __init_subclass__(cls, **kwargs: dict[str, Any]) -> None:
"""Setup unique state for lock subclasses."""
super().__init_subclass__(**kwargs)
cls._instances = WeakValueDictionary()
cls._instances_lock = Lock()
def __init__( # noqa: PLR0913
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = True, # noqa: FBT001, FBT002
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
) -> None:
"""
Create a new lock object.
:param lock_file: path to the file
:param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the
acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a
negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock.
:param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and
default ACLs, preserving POSIX default ACL inheritance in shared directories.
:param thread_local: Whether this object's internal context should be thread local or not. If this is set to
``False`` then the lock will be reentrant across threads. When ``True`` (the default), **all fields of the
lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``,
``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change
the value seen by another thread; threads that did not perform the write continue to see the value supplied
at construction time. If you need configuration values to be visible across threads, construct the lock
with ``thread_local=False``.
:param blocking: whether the lock should be blocking or not
:param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock
file. This is useful if you want to use the lock object for reentrant locking without needing to pass the
same object around.
:param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value
in the acquire method, if no poll_interval value (``None``) is given.
:param lifetime: maximum time in seconds a lock can be held before it is considered expired. When set, a waiting
process will break a lock whose file modification time is older than ``lifetime`` seconds. ``None`` (the
default) means locks never expire.
"""
self._is_thread_local = thread_local
self._is_singleton = is_singleton
# Create the context. Note that external code should not work with the context directly and should instead use
# properties of this class.
kwargs: dict[str, Any] = {
"lock_file": os.fspath(lock_file),
"timeout": timeout,
"mode": mode,
"blocking": blocking,
"poll_interval": poll_interval,
"lifetime": lifetime,
}
self._context: FileLockContext = (ThreadLocalFileContext if thread_local else FileLockContext)(**kwargs)
def is_thread_local(self) -> bool:
""":returns: a flag indicating if this lock is thread local or not"""
return self._is_thread_local
@property
def is_singleton(self) -> bool:
"""
A flag indicating if this lock is singleton or not.
.. versionadded:: 3.13.0
"""
return self._is_singleton
@property
def lock_file(self) -> str:
"""Path to the lock file."""
return self._context.lock_file
@property
def timeout(self) -> float:
"""
The default timeout value, in seconds.
.. versionadded:: 2.0.0
"""
return self._context.timeout
@timeout.setter
def timeout(self, value: float | str) -> None:
"""
Change the default timeout value.
:param value: the new value, in seconds
"""
self._context.timeout = float(value)
@property
def blocking(self) -> bool:
"""
Whether the locking is blocking or not.
.. versionadded:: 3.14.0
"""
return self._context.blocking
@blocking.setter
def blocking(self, value: bool) -> None:
"""
Change the default blocking value.
:param value: the new value as bool
"""
self._context.blocking = value
@property
def poll_interval(self) -> float:
"""
The default polling interval, in seconds.
.. versionadded:: 3.24.0
"""
return self._context.poll_interval
@poll_interval.setter
def poll_interval(self, value: float) -> None:
"""
Change the default polling interval.
:param value: the new value, in seconds
"""
self._context.poll_interval = value
@property
def lifetime(self) -> float | None:
"""
The lock lifetime in seconds, or ``None`` if the lock never expires.
.. versionadded:: 3.24.0
"""
return self._context.lifetime
@lifetime.setter
def lifetime(self, value: float | None) -> None:
"""
Change the lock lifetime.
:param value: the new value in seconds, or ``None`` to disable expiration
:raises ValueError: if *value* is a negative number
:raises TypeError: if *value* is not ``None`` and not a real number
"""
if value is not None:
if isinstance(value, bool) or not isinstance(value, (int, float)):
msg = f"lifetime must be a non-negative number or None, not {type(value).__name__}"
raise TypeError(msg)
if value < 0:
msg = f"lifetime must be non-negative, not {value!r}"
raise ValueError(msg)
self._context.lifetime = value
@property
def mode(self) -> int:
"""The file permissions for the lockfile."""
return 0o644 if self._context.mode == _UNSET_FILE_MODE else self._context.mode
@property
def has_explicit_mode(self) -> bool:
"""Whether the file permissions were explicitly set."""
return self._context.mode != _UNSET_FILE_MODE
def _open_mode(self) -> int:
""":returns: the mode for os.open() — 0o666 when unset (let umask/ACLs decide), else the explicit mode"""
return 0o666 if self._context.mode == _UNSET_FILE_MODE else self._context.mode
def _try_break_expired_lock(self) -> None:
"""Remove the lock file if its modification time exceeds the configured :attr:`lifetime`."""
if (lifetime := self._context.lifetime) is None:
return
with contextlib.suppress(OSError):
# lstat, not stat: an attacker with write access to the lock directory can replace a held
# lock file with a symlink pointing at an old file, making stat() report the target's stale
# mtime so a waiter breaks a live lock and two processes hold it at once. lstat reads the
# symlink's own mtime, matching the O_NOFOLLOW reads elsewhere.
st = os.lstat(self.lock_file)
if time.time() - st.st_mtime < lifetime:
return
break_lock_file(self.lock_file, st.st_mtime, st.st_ino)
@abstractmethod
def _acquire(self) -> None:
"""If the file lock could be acquired, self._context.lock_file_fd holds the file descriptor of the lock file."""
raise NotImplementedError
@abstractmethod
def _release(self) -> None:
"""Releases the lock and sets self._context.lock_file_fd to None."""
raise NotImplementedError
@property
def is_locked(self) -> bool:
"""
A boolean indicating if the lock file is holding the lock currently.
.. versionchanged:: 2.0.0
This was previously a method and is now a property.
"""
return self._context.lock_file_fd is not None
@property
def lock_counter(self) -> int:
"""The number of times this lock has been acquired (but not yet released)."""
return self._context.lock_counter
@staticmethod
def _check_give_up( # noqa: PLR0913
lock_id: int,
lock_filename: str,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
timeout: float,
start_time: float,
) -> bool:
if blocking is False:
_LOGGER.debug("Failed to immediately acquire lock %s on %s", lock_id, lock_filename)
return True
if cancel_check is not None and cancel_check():
_LOGGER.debug("Cancellation requested for lock %s on %s", lock_id, lock_filename)
return True
if 0 <= timeout < time.perf_counter() - start_time:
_LOGGER.debug("Timeout on acquiring lock %s on %s", lock_id, lock_filename)
return True
return False
def acquire(
self,
timeout: float | None = None,
poll_interval: float | None = None,
*,
poll_intervall: float | None = None,
blocking: bool | None = None,
cancel_check: Callable[[], bool] | None = None,
) -> AcquireReturnProxy:
"""
Try to acquire the file lock.
:param timeout: maximum wait time for acquiring the lock, ``None`` means use the default :attr:`~timeout` is and
if ``timeout < 0``, there is no timeout and this method will block until the lock could be acquired
:param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default
:attr:`~poll_interval`
:param poll_intervall: deprecated, kept for backwards compatibility, use ``poll_interval`` instead
:param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the
first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired.
:param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll
iteration. When triggered, raises :class:`~Timeout` just like an expired timeout.
:returns: a context object that will unlock the file when the context is exited
:raises Timeout: if fails to acquire lock within the timeout period
.. code-block:: python
# You can use this method in the context manager (recommended)
with lock.acquire():
pass
# Or use an equivalent try-finally construct:
lock.acquire()
try:
pass
finally:
lock.release()
.. versionchanged:: 2.0.0
This method returns now a *proxy* object instead of *self*, so that it can be used in a with statement
without side effects.
"""
# Use the default timeout, if no timeout is provided.
if timeout is None:
timeout = self._context.timeout
if blocking is None:
blocking = self._context.blocking
if poll_intervall is not None:
msg = "use poll_interval instead of poll_intervall"
warnings.warn(msg, DeprecationWarning, stacklevel=2)
poll_interval = poll_intervall
poll_interval = poll_interval if poll_interval is not None else self._context.poll_interval
# Increment the number right at the beginning. We can still undo it, if something fails.
self._context.lock_counter += 1
canonical = _canonical(self.lock_file)
self._raise_if_would_deadlock(canonical, timeout=timeout, blocking=blocking)
start_time = time.perf_counter()
try:
self._poll_until_acquired(
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
poll_interval=poll_interval,
start_time=start_time,
)
except BaseException:
self._undo_acquire(canonical)
raise
self._commit_acquire(canonical)
return AcquireReturnProxy(lock=self)
def _raise_if_would_deadlock(self, canonical: str, *, timeout: float, blocking: bool) -> None:
"""
Fail fast when a *different* live instance already holds this path on the current thread/task.
Only the first, indefinitely-blocking acquire can self-deadlock this way: waiting in the OS primitive would
block on a lock this thread already owns. A finite timeout or ``blocking=False`` keeps the normal Timeout path.
"""
would_block = self._context.lock_counter == 1 and not self.is_locked and timeout < 0 and blocking
if would_block and _registry.held.get(canonical) not in {None, id(self)}:
self._context.lock_counter -= 1
msg = (
f"Deadlock: lock '{self.lock_file}' is already held by a different {self._deadlock_holder_desc}. "
f"Use is_singleton=True to enable reentrant locking across instances."
)
raise RuntimeError(msg)
def _undo_acquire(self, canonical: str) -> None:
"""Roll back the counter after a failed acquire, dropping the registry entry once nothing holds the path."""
self._context.lock_counter = max(0, self._context.lock_counter - 1)
if self._context.lock_counter == 0:
_registry.held.pop(canonical, None)
def _commit_acquire(self, canonical: str) -> None:
"""Record this instance as the holder once the first acquire succeeds, so peers can detect the deadlock."""
if self._context.lock_counter == 1:
_registry.held[canonical] = id(self)
def _drop_registry_entry(self) -> None:
"""Forget this path's holder on release so a later cross-instance acquire is not misread as a deadlock."""
_registry.held.pop(_canonical(self.lock_file), None)
def _poll_until_acquired(
self,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
timeout: float,
poll_interval: float,
start_time: float,
) -> None:
lock_id = id(self)
lock_filename = self.lock_file
while True:
if not self.is_locked:
self._try_break_expired_lock()
_LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename)
self._acquire()
if self.is_locked:
_LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename)
return
if self._check_give_up(
lock_id,
lock_filename,
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
start_time=start_time,
):
raise Timeout(lock_filename)
msg = "Lock %s not acquired on %s, waiting %s seconds ..."
_LOGGER.debug(msg, lock_id, lock_filename, poll_interval)
time.sleep(poll_interval)
def release(self, force: bool = False) -> None: # noqa: FBT001, FBT002
"""
Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file
itself may be deleted automatically, the behavior is platform-specific.
:param force: If true, the lock counter is ignored and the lock is released in every case.
"""
if self.is_locked:
self._context.lock_counter -= 1
if self._context.lock_counter == 0 or force:
lock_id, lock_filename = id(self), self.lock_file
_LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename)
self._release()
self._context.lock_counter = 0
self._drop_registry_entry()
_LOGGER.debug("Lock %s released on %s", lock_id, lock_filename)
def __enter__(self) -> Self:
"""
Acquire the lock.
:returns: the lock object
"""
self.acquire()
return self
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
"""
Release the lock.
:param exc_type: the exception type if raised
:param exc_value: the exception value if raised
:param traceback: the exception traceback if raised
"""
self.release()
def __del__(self) -> None:
"""Called when the lock object is deleted."""
self.release(force=True)
__all__ = [
"_UNSET_FILE_MODE",
"AcquireReturnProxy",
"BaseFileLock",
]
@@ -0,0 +1,216 @@
"""Async wrapper around :class:`ReadWriteLock` for use with ``asyncio``."""
from __future__ import annotations
import asyncio
import functools
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING
from ._read_write import ReadWriteLock
if TYPE_CHECKING:
import os
from collections.abc import AsyncGenerator, Callable
from concurrent import futures
from types import TracebackType
class AsyncAcquireReadWriteReturnProxy:
"""Context-aware object that releases the async read/write lock on exit."""
def __init__(self, lock: AsyncReadWriteLock) -> None:
self.lock = lock
async def __aenter__(self) -> AsyncReadWriteLock:
return self.lock
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
await self.lock.release()
class AsyncReadWriteLock:
"""
Async wrapper around :class:`ReadWriteLock` for use in ``asyncio`` applications.
Because Python's :mod:`sqlite3` module has no async API, all blocking SQLite operations are dispatched to a thread
pool via ``loop.run_in_executor()``. Reentrancy, upgrade/downgrade rules, and singleton behavior are delegated
to the underlying :class:`ReadWriteLock`.
:param lock_file: path to the SQLite database file used as the lock
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:param is_singleton: if ``True``, reuse existing :class:`ReadWriteLock` instances for the same resolved path
:param loop: event loop for ``run_in_executor``; ``None`` uses the running loop
:param executor: executor for ``run_in_executor``. When ``None`` a dedicated single-thread executor is created
and owned by this lock, ensuring every operation runs on the same thread (required for SQLite affinity); it
is shut down by :meth:`close`. A caller-supplied executor is used as-is and never shut down here, so when no
executor is passed remember to call :meth:`close` to release the owned one.
.. versionadded:: 3.21.0
"""
def __init__( # noqa: PLR0913
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
loop: asyncio.AbstractEventLoop | None = None,
executor: futures.Executor | None = None,
) -> None:
self._lock = ReadWriteLock(lock_file, timeout, blocking=blocking, is_singleton=is_singleton)
self._loop = loop
self._owns_executor = executor is None
self._executor = executor or ThreadPoolExecutor(max_workers=1)
@property
def lock_file(self) -> str:
"""The path to the lock file."""
return self._lock.lock_file
@property
def timeout(self) -> float:
"""The default timeout."""
return self._lock.timeout
@property
def blocking(self) -> bool:
"""Whether blocking is enabled by default."""
return self._lock.blocking
@property
def loop(self) -> asyncio.AbstractEventLoop | None:
"""The event loop (or ``None`` for the running loop)."""
return self._loop
@property
def executor(self) -> futures.Executor:
"""The executor used for ``run_in_executor`` (a dedicated single-thread one if none was supplied)."""
return self._executor
async def _run(self, func: Callable[..., object], *args: object, **kwargs: object) -> object:
loop = self._loop or asyncio.get_running_loop()
return await loop.run_in_executor(self._executor, functools.partial(func, *args, **kwargs))
async def acquire_read(self, timeout: float = -1, *, blocking: bool = True) -> AsyncAcquireReadWriteReturnProxy:
"""
Acquire a shared read lock.
See :meth:`ReadWriteLock.acquire_read` for full semantics.
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: a proxy that can be used as an async context manager to release the lock
:raises RuntimeError: if a write lock is already held on this instance
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self._run(self._lock.acquire_read, timeout, blocking=blocking)
return AsyncAcquireReadWriteReturnProxy(lock=self)
async def acquire_write(self, timeout: float = -1, *, blocking: bool = True) -> AsyncAcquireReadWriteReturnProxy:
"""
Acquire an exclusive write lock.
See :meth:`ReadWriteLock.acquire_write` for full semantics.
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: a proxy that can be used as an async context manager to release the lock
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self._run(self._lock.acquire_write, timeout, blocking=blocking)
return AsyncAcquireReadWriteReturnProxy(lock=self)
async def release(self, *, force: bool = False) -> None:
"""
Release one level of the current lock.
See :meth:`ReadWriteLock.release` for full semantics.
:param force: if ``True``, release the lock completely regardless of the current lock level
:raises RuntimeError: if no lock is currently held and *force* is ``False``
"""
await self._run(self._lock.release, force=force)
@asynccontextmanager
async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases a shared read lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
if timeout is None:
timeout = self._lock.timeout
if blocking is None:
blocking = self._lock.blocking
await self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
@asynccontextmanager
async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases an exclusive write lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
if timeout is None:
timeout = self._lock.timeout
if blocking is None:
blocking = self._lock.blocking
await self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
async def close(self) -> None:
"""
Release the lock (if held) and close the underlying SQLite connection.
After calling this method, the lock instance is no longer usable.
"""
await self._run(self._lock.close)
if self._owns_executor:
self._executor.shutdown(wait=False)
def __del__(self) -> None:
# Safety net: if close() was never called, still shut down the executor we created so its worker thread does
# not outlive the lock. A caller-supplied executor is left untouched. shutdown(wait=False) never blocks.
if getattr(self, "_owns_executor", False):
self._executor.shutdown(wait=False)
__all__ = [
"AsyncAcquireReadWriteReturnProxy",
"AsyncReadWriteLock",
]
@@ -0,0 +1,28 @@
from __future__ import annotations
class Timeout(TimeoutError): # noqa: N818
"""Raised when the lock could not be acquired in *timeout* seconds."""
def __init__(self, lock_file: str) -> None:
super().__init__()
self._lock_file = lock_file
def __reduce__(self) -> tuple[type[Timeout], tuple[str]]:
return self.__class__, (self._lock_file,) # Properly pickle the exception
def __str__(self) -> str:
return f"The file lock '{self._lock_file}' could not be acquired."
def __repr__(self) -> str:
return f"{self.__class__.__name__}({self.lock_file!r})"
@property
def lock_file(self) -> str:
"""The path of the file lock."""
return self._lock_file
__all__ = [
"Timeout",
]
@@ -0,0 +1,385 @@
from __future__ import annotations
import atexit
import logging
import os
import pathlib
import sqlite3
import threading
import time
from contextlib import contextmanager, suppress
from typing import TYPE_CHECKING, Literal
from weakref import WeakValueDictionary
from ._api import AcquireReturnProxy
from ._error import Timeout
if TYPE_CHECKING:
from collections.abc import Generator
_LOGGER = logging.getLogger("filelock")
_all_connections: set[sqlite3.Connection] = set()
_all_connections_lock = threading.Lock()
def _cleanup_connections() -> None:
with _all_connections_lock:
for con in list(_all_connections):
with suppress(Exception):
con.close()
_all_connections.clear()
atexit.register(_cleanup_connections)
# sqlite3_busy_timeout() accepts a C int, max 2_147_483_647 on 32-bit. Use a lower value to be safe (~23 days).
_MAX_SQLITE_TIMEOUT_MS = 2_000_000_000 - 1
def timeout_for_sqlite(timeout: float, *, blocking: bool, already_waited: float) -> int:
if blocking is False:
return 0
if timeout == -1:
return _MAX_SQLITE_TIMEOUT_MS
if timeout < 0:
msg = "timeout must be a non-negative number or -1"
raise ValueError(msg)
remaining = max(timeout - already_waited, 0) if timeout > 0 else timeout
timeout_ms = int(remaining * 1000)
if timeout_ms > _MAX_SQLITE_TIMEOUT_MS or timeout_ms < 0:
_LOGGER.warning("timeout %s is too large for SQLite, using %s ms instead", timeout, _MAX_SQLITE_TIMEOUT_MS)
return _MAX_SQLITE_TIMEOUT_MS
return timeout_ms
class _ReadWriteLockMeta(type):
"""
Metaclass that handles singleton resolution when is_singleton=True.
Singleton logic lives here rather than in ReadWriteLock.get_lock so that ``ReadWriteLock(path)`` transparently
returns cached instances without a 2-arg ``super()`` call that type checkers cannot verify.
"""
_instances: WeakValueDictionary[pathlib.Path, ReadWriteLock]
_instances_lock: threading.Lock
def __call__(
cls,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
) -> ReadWriteLock:
if not is_singleton:
return super().__call__(lock_file, timeout, blocking=blocking, is_singleton=is_singleton)
normalized = pathlib.Path(lock_file).resolve()
with cls._instances_lock:
if normalized not in cls._instances:
instance = super().__call__(lock_file, timeout, blocking=blocking, is_singleton=is_singleton)
cls._instances[normalized] = instance
else:
instance = cls._instances[normalized]
if instance.timeout != timeout or instance.blocking != blocking:
msg = (
f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking},"
f" cannot be changed to timeout={timeout}, blocking={blocking}"
)
raise ValueError(msg)
return instance
class ReadWriteLock(metaclass=_ReadWriteLockMeta):
"""
Cross-process read-write lock backed by SQLite.
Allows concurrent shared readers or a single exclusive writer. The lock is reentrant within the same mode (multiple
``acquire_read`` calls nest, as do multiple ``acquire_write`` calls from the same thread), but upgrading from read
to write or downgrading from write to read raises :class:`RuntimeError`. Write locks are pinned to the thread that
acquired them.
By default, ``is_singleton=True``: calling ``ReadWriteLock(path)`` with the same resolved path returns the same
instance. The lock file must use a ``.db`` extension (SQLite database).
:param lock_file: path to the SQLite database file used as the lock
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:param is_singleton: if ``True``, reuse existing instances for the same resolved path
.. versionadded:: 3.21.0
"""
_instances: WeakValueDictionary[pathlib.Path, ReadWriteLock] = WeakValueDictionary()
_instances_lock = threading.Lock()
@classmethod
def get_lock(
cls, lock_file: str | os.PathLike[str], timeout: float = -1, *, blocking: bool = True
) -> ReadWriteLock:
"""
Return the singleton :class:`ReadWriteLock` for *lock_file*.
:param lock_file: path to the SQLite database file used as the lock
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: the singleton lock instance
:raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values
"""
return cls(lock_file, timeout, blocking=blocking)
def __init__(
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True, # noqa: ARG002 # consumed by _ReadWriteLockMeta.__call__
) -> None:
self.lock_file = os.fspath(lock_file)
self.timeout = timeout
self.blocking = blocking
self._transaction_lock = threading.Lock() # serializes the (possibly blocking) SQLite transaction work
self._internal_lock = threading.Lock() # protects _lock_level / _current_mode updates and rollback
self._lock_level = 0
self._current_mode: Literal["read", "write"] | None = None
self._write_thread_id: int | None = None
self._con = sqlite3.connect(self.lock_file, check_same_thread=False)
with _all_connections_lock:
_all_connections.add(self._con)
def _acquire_transaction_lock(self, *, blocking: bool, timeout: float) -> None:
if not blocking:
acquired = self._transaction_lock.acquire(blocking=False)
elif timeout == -1:
acquired = self._transaction_lock.acquire(blocking=True)
else:
acquired = self._transaction_lock.acquire(blocking=True, timeout=timeout)
if not acquired:
raise Timeout(self.lock_file) from None
def _validate_reentrant(self, mode: Literal["read", "write"]) -> AcquireReturnProxy:
opposite = "write" if mode == "read" else "read"
direction = "downgrade" if mode == "read" else "upgrade"
if self._current_mode != mode:
msg = (
f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): "
f"already holding a {opposite} lock ({direction} not allowed)"
)
raise RuntimeError(msg)
if mode == "write" and (cur := threading.get_ident()) != self._write_thread_id:
msg = (
f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) "
f"from thread {cur} while it is held by thread {self._write_thread_id}"
)
raise RuntimeError(msg)
self._lock_level += 1
return AcquireReturnProxy(lock=self)
def _configure_and_begin(
self, mode: Literal["read", "write"], timeout: float, *, blocking: bool, start_time: float
) -> None:
waited = time.perf_counter() - start_time
timeout_ms = timeout_for_sqlite(timeout, blocking=blocking, already_waited=waited)
self._con.execute(f"PRAGMA busy_timeout={timeout_ms};").close()
# Use legacy journal mode (not WAL) because WAL does not block readers when a concurrent EXCLUSIVE
# write transaction is active, making read-write locking impossible without modifying table data.
# MEMORY is safe here since no actual writes happen — crashes cannot corrupt the DB.
# See https://sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions
#
# Set here (not in __init__) because this pragma itself may block on a locked database,
# so it must run after busy_timeout is configured above.
self._con.execute("PRAGMA journal_mode=MEMORY;").close()
# Recompute remaining timeout after the potentially blocking journal_mode pragma.
waited = time.perf_counter() - start_time
if (recomputed := timeout_for_sqlite(timeout, blocking=blocking, already_waited=waited)) != timeout_ms:
self._con.execute(f"PRAGMA busy_timeout={recomputed};").close()
stmt = "BEGIN EXCLUSIVE TRANSACTION;" if mode == "write" else "BEGIN TRANSACTION;"
self._con.execute(stmt).close()
if mode == "read":
# A SELECT is needed to force SQLite to actually acquire the SHARED lock on the database.
# https://www.sqlite.org/lockingv3.html#transaction_control
self._con.execute("SELECT name FROM sqlite_schema LIMIT 1;").close()
def _acquire(self, mode: Literal["read", "write"], timeout: float, *, blocking: bool) -> AcquireReturnProxy:
with self._internal_lock:
if self._lock_level > 0:
return self._validate_reentrant(mode)
start_time = time.perf_counter()
self._acquire_transaction_lock(blocking=blocking, timeout=timeout)
try:
return self._do_acquire_inner(mode, timeout, blocking=blocking, start_time=start_time)
except sqlite3.Error as exc:
# A read acquire runs BEGIN (a deferred transaction that takes no database lock) and only then the
# SELECT that actually takes the SHARED lock. If a writer grabs the EXCLUSIVE lock between the two,
# the SELECT fails but BEGIN's transaction is left open on the shared connection. Roll it back here,
# while we still hold _transaction_lock, otherwise the next acquire's BEGIN dies with "cannot start a
# transaction within a transaction" and the instance is wedged for good. Catch only sqlite3.Error: a
# reentrant-validation RuntimeError from _do_acquire_inner's double-check must not roll back, or it
# would drop a transaction another thread legitimately holds on the shared connection.
with suppress(sqlite3.Error):
self._con.rollback()
if isinstance(exc, sqlite3.OperationalError) and "database is locked" in str(exc):
raise Timeout(self.lock_file) from None
raise
finally:
self._transaction_lock.release()
def _do_acquire_inner(
self,
mode: Literal["read", "write"],
timeout: float,
*,
blocking: bool,
start_time: float,
) -> AcquireReturnProxy:
# Double-check: another thread may have acquired the lock while we waited on _transaction_lock.
with self._internal_lock:
if self._lock_level > 0:
return self._validate_reentrant(mode)
self._configure_and_begin(mode, timeout, blocking=blocking, start_time=start_time)
with self._internal_lock:
self._current_mode = mode
self._lock_level = 1
if mode == "write":
self._write_thread_id = threading.get_ident()
return AcquireReturnProxy(lock=self)
def acquire_read(self, timeout: float = -1, *, blocking: bool = True) -> AcquireReturnProxy:
"""
Acquire a shared read lock.
If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a
read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed).
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a write lock is already held on this instance
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
return self._acquire("read", timeout, blocking=blocking)
def acquire_write(self, timeout: float = -1, *, blocking: bool = True) -> AcquireReturnProxy:
"""
Acquire an exclusive write lock.
If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant).
Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not allowed).
Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises
:class:`RuntimeError`.
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
return self._acquire("write", timeout, blocking=blocking)
def release(self, *, force: bool = False) -> None:
"""
Release one level of the current lock.
When the lock level reaches zero the underlying SQLite transaction is rolled back, releasing the database lock.
:param force: if ``True``, release the lock completely regardless of the current lock level
:raises RuntimeError: if no lock is currently held and *force* is ``False``
"""
should_rollback = False
with self._internal_lock:
if self._lock_level == 0:
if force:
return
msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held"
raise RuntimeError(msg)
if force:
self._lock_level = 0
else:
self._lock_level -= 1
if self._lock_level == 0:
self._current_mode = None
self._write_thread_id = None
should_rollback = True
if should_rollback:
# The rollback ends the transaction on the shared connection, so it has to be serialized against
# acquire()'s BEGIN the same way acquire() already serializes itself with _transaction_lock. Without
# this, another thread that sees lock_level back at 0 can start its BEGIN while this rollback's
# transaction is still open (raising "cannot start a transaction within a transaction") or, in the
# other ordering, have its freshly started transaction rolled back here, dropping the database lock
# while it still believes it holds it.
with self._transaction_lock:
self._con.rollback()
@contextmanager
def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases a shared read lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
if timeout is None:
timeout = self.timeout
if blocking is None:
blocking = self.blocking
self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
self.release()
@contextmanager
def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases an exclusive write lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
if timeout is None:
timeout = self.timeout
if blocking is None:
blocking = self.blocking
self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
self.release()
def close(self) -> None:
"""
Release the lock (if held) and close the underlying SQLite connection.
After calling this method, the lock instance is no longer usable.
"""
self.release(force=True)
self._con.close()
with _all_connections_lock:
_all_connections.discard(self._con)
@@ -0,0 +1,243 @@
from __future__ import annotations
import os
import socket
import sys
import time
from contextlib import suppress
from errno import EACCES, EEXIST, EPERM, ESRCH
from pathlib import Path
from ._api import BaseFileLock
from ._util import break_lock_file, ensure_directory_exists, raise_on_not_writable_file
_WIN_SYNCHRONIZE = 0x100000
_WIN_ERROR_INVALID_PARAMETER = 87
_WIN_PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
_MALFORMED_LOCK_AGE_THRESHOLD = 2.0
_MAX_LOCK_FILE_SIZE = 1024
class SoftFileLock(BaseFileLock):
"""
Portable file lock based on file existence.
Unlike :class:`UnixFileLock <filelock.UnixFileLock>` and :class:`WindowsFileLock <filelock.WindowsFileLock>`, this
lock does not use OS-level locking primitives. Instead, it creates the lock file with ``O_CREAT | O_EXCL`` and
treats its existence as the lock indicator. This makes it work on any filesystem but leaves stale lock files behind
if the process crashes without releasing the lock.
To mitigate stale locks, the lock file contains the PID and hostname of the holding process. On contention, if the
holder is on the same host and its PID no longer exists, the stale lock is broken automatically.
"""
def _acquire(self) -> None:
raise_on_not_writable_file(self.lock_file)
ensure_directory_exists(self.lock_file)
flags = (
os.O_WRONLY # open for writing only
| os.O_CREAT
| os.O_EXCL # together with above raise EEXIST if the file specified by filename exists
| os.O_TRUNC # truncate the file to zero byte
)
if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None:
flags |= o_nofollow
try:
file_handler = os.open(self.lock_file, flags, self._open_mode())
except OSError as exception:
if not (
exception.errno == EEXIST or (exception.errno == EACCES and sys.platform == "win32")
): # pragma: win32 no cover
raise
self._try_break_stale_lock()
else:
self._write_lock_info(file_handler)
self._context.lock_file_fd = file_handler
def _try_break_stale_lock(self) -> None:
with suppress(OSError, ValueError):
content, mtime, ino = _read_lock_file(self.lock_file)
holder = _parse_lock_holder(content)
if holder is None:
# Unparsable: wrong line count, a non-integer PID or creation time, empty, oversized or not UTF-8.
# Self-heal only once the file is clearly not a half-written fresh lock (a peer between O_EXCL and
# _write_lock_info), so the brief create-then-write window is never mistaken for a stale lock.
if time.time() - mtime >= _MALFORMED_LOCK_AGE_THRESHOLD:
break_lock_file(self.lock_file, mtime, ino)
return
pid, hostname, creation_time = holder
if hostname != socket.gethostname():
return
if self._is_process_alive(pid):
if sys.platform != "win32" or creation_time is None: # pragma: win32 no cover
return # same process, or no creation time to disambiguate a recycled PID — don't evict
actual = self._get_process_creation_time(pid) # pragma: win32 cover
if actual is None or actual == creation_time: # pragma: win32 cover
return # same process or can't verify — don't evict
# else: PID alive but creation time differs — the PID was recycled, so the lock is stale.
break_lock_file(self.lock_file, mtime, ino)
@staticmethod
def _is_process_alive(pid: int) -> bool:
if sys.platform == "win32": # pragma: win32 cover
import ctypes # noqa: PLC0415
kernel32 = ctypes.windll.kernel32
handle = kernel32.OpenProcess(_WIN_SYNCHRONIZE, 0, pid)
if handle:
kernel32.CloseHandle(handle)
return True
return kernel32.GetLastError() != _WIN_ERROR_INVALID_PARAMETER
try:
os.kill(pid, 0)
except OSError as exc:
if exc.errno == ESRCH:
return False
if exc.errno == EPERM:
return True
raise
return True
@staticmethod
def _get_process_creation_time(pid: int) -> int | None:
"""Return the process creation FILETIME as an integer on Windows, ``None`` otherwise."""
if sys.platform != "win32": # pragma: win32 no cover
return None
import ctypes # pragma: win32 cover # noqa: PLC0415
from ctypes import wintypes # noqa: PLC0415
kernel32 = ctypes.windll.kernel32
handle = kernel32.OpenProcess(_WIN_PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
if not handle:
return None
try:
creation = wintypes.FILETIME()
exit_time = wintypes.FILETIME()
kernel_time = wintypes.FILETIME()
user_time = wintypes.FILETIME()
if not kernel32.GetProcessTimes(
handle,
ctypes.byref(creation),
ctypes.byref(exit_time),
ctypes.byref(kernel_time),
ctypes.byref(user_time),
):
return None
finally:
kernel32.CloseHandle(handle)
return (creation.dwHighDateTime << 32) | creation.dwLowDateTime
@staticmethod
def _write_lock_info(fd: int) -> None:
with suppress(OSError):
info = f"{os.getpid()}\n{socket.gethostname()}\n"
if sys.platform == "win32" and (ct := SoftFileLock._get_process_creation_time(os.getpid())) is not None:
info += f"{ct}\n"
os.write(fd, info.encode())
@property
def pid(self) -> int | None:
"""
The PID of the process holding this lock, read from the lock file.
:returns: the PID as an integer, or ``None`` if the lock file does not exist or cannot be parsed
"""
with suppress(OSError, ValueError):
holder = _parse_lock_holder(_read_lock_file(self.lock_file)[0])
if holder is not None:
return holder[0]
return None
@property
def is_lock_held_by_us(self) -> bool:
"""
Whether this lock is held by the current process.
:returns: ``True`` if the lock file exists and names the current process's PID and hostname
"""
with suppress(OSError, ValueError):
holder = _parse_lock_holder(_read_lock_file(self.lock_file)[0])
if holder is not None:
pid, hostname, _ = holder
return pid == os.getpid() and hostname == socket.gethostname()
return False
def break_lock(self) -> None:
"""Forcibly break the lock by removing the lock file, regardless of who holds it."""
with suppress(OSError):
Path(self.lock_file).unlink()
def _release(self) -> None:
assert self._context.lock_file_fd is not None # noqa: S101
os.close(self._context.lock_file_fd)
self._context.lock_file_fd = None
if sys.platform == "win32":
self._windows_unlink_with_retry()
else:
with suppress(OSError):
Path(self.lock_file).unlink()
def _windows_unlink_with_retry(self) -> None:
max_retries = 10
retry_delay = 0.001
for attempt in range(max_retries):
# Windows doesn't immediately release file handles after close, causing EACCES/EPERM on unlink
try:
Path(self.lock_file).unlink()
except OSError as exc: # noqa: PERF203
if exc.errno not in {EACCES, EPERM}:
return
if attempt < max_retries - 1:
time.sleep(retry_delay)
retry_delay *= 2
else:
return
def _read_lock_file(path: str) -> tuple[str | None, float, int]:
# The lock file is created with O_EXCL | O_NOFOLLOW, so a symlink here is a hostile replacement and must
# not be followed. O_NONBLOCK keeps an attacker-placed FIFO from stalling the open (O_NOFOLLOW alone only
# rejects a symlink, not a real FIFO at the path), and the capped read stops a huge file (e.g. /dev/zero)
# from exhausting memory. Content is None when the file is too large or not UTF-8, but the mtime and inode
# still flow back so the caller can evict it as a stale, malformed lock and verify identity before breaking.
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
try:
st, data = os.fstat(fd), os.read(fd, _MAX_LOCK_FILE_SIZE + 1)
finally:
os.close(fd)
if len(data) <= _MAX_LOCK_FILE_SIZE:
with suppress(UnicodeDecodeError):
return data.decode("utf-8"), st.st_mtime, st.st_ino
return None, st.st_mtime, st.st_ino
def _parse_lock_holder(content: str | None) -> tuple[int, str, int | None] | None:
# A well-formed lock file is "<pid>\n<hostname>\n" with an optional "<creation_time>\n" third line on Windows.
# Anything else — wrong line count, a non-integer PID or creation time, empty or unreadable content — is
# unparsable; returning None lets the caller treat it as a malformed lock to self-heal rather than a holder.
if not content or len(lines := content.strip().splitlines()) not in {2, 3}:
return None
try:
pid = int(lines[0])
creation_time = int(lines[2]) if len(lines) == 3 else None # noqa: PLR2004
except ValueError:
return None
# A pid outside the valid range is a malformed lock, not a holder. Without this, a non-positive pid
# reaches os.kill() where 0 / -1 mean "the caller's own process group / every process" so a dead
# holder reads as alive and the lock is never reclaimed, while an oversized pid raises OverflowError
# (not OSError/ValueError) out of the self-heal path. _parse_marker_bytes already enforces this range.
if not 1 <= pid <= 2**31 - 1:
return None
return pid, lines[1], creation_time
__all__ = [
"SoftFileLock",
]
@@ -0,0 +1,12 @@
"""Cross-process and cross-host reader/writer lock on :class:`~filelock.SoftFileLock` primitives."""
from __future__ import annotations
from ._async import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock
from ._sync import SoftReadWriteLock
__all__ = [
"AsyncAcquireSoftReadWriteReturnProxy",
"AsyncSoftReadWriteLock",
"SoftReadWriteLock",
]
@@ -0,0 +1,213 @@
"""Async wrapper around :class:`SoftReadWriteLock` for use with ``asyncio``."""
from __future__ import annotations
import asyncio
import functools
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING
from ._sync import SoftReadWriteLock
if TYPE_CHECKING:
import os
from collections.abc import AsyncGenerator, Callable
from concurrent import futures
from types import TracebackType
class AsyncAcquireSoftReadWriteReturnProxy:
"""Async context-aware object that releases an :class:`AsyncSoftReadWriteLock` on exit."""
def __init__(self, lock: AsyncSoftReadWriteLock) -> None:
self.lock = lock
async def __aenter__(self) -> AsyncSoftReadWriteLock:
return self.lock
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
await self.lock.release()
class AsyncSoftReadWriteLock:
"""
Async wrapper around :class:`SoftReadWriteLock` for ``asyncio`` applications.
The sync class's blocking filesystem operations run on a thread pool via ``loop.run_in_executor()``.
Reentrancy, upgrade/downgrade rules, fork handling, heartbeat and TTL stale detection, and singleton
behavior are delegated to the underlying :class:`SoftReadWriteLock`.
:param lock_file: path to the lock file; sidecar state/write/readers live next to it
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
:param is_singleton: if ``True``, reuse existing :class:`SoftReadWriteLock` instances per resolved path
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
:param stale_threshold: seconds of mtime inactivity before a marker is stale; defaults to ``3 * heartbeat_interval``
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
:param loop: event loop for ``run_in_executor``; ``None`` uses the running loop
:param executor: executor for ``run_in_executor``; ``None`` uses the default executor
.. versionadded:: 3.27.0
"""
def __init__( # noqa: PLR0913
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
heartbeat_interval: float = 30.0,
stale_threshold: float | None = None,
poll_interval: float = 0.25,
loop: asyncio.AbstractEventLoop | None = None,
executor: futures.Executor | None = None,
) -> None:
self._lock = SoftReadWriteLock(
lock_file,
timeout,
blocking=blocking,
is_singleton=is_singleton,
heartbeat_interval=heartbeat_interval,
stale_threshold=stale_threshold,
poll_interval=poll_interval,
)
self._loop = loop
self._executor = executor
@property
def lock_file(self) -> str:
"""The path to the lock file passed to the constructor."""
return self._lock.lock_file
@property
def timeout(self) -> float:
"""The default timeout applied when ``acquire_read`` / ``acquire_write`` is called without one."""
return self._lock.timeout
@property
def blocking(self) -> bool:
"""Whether ``acquire_*`` defaults to blocking; ``False`` makes contention raise immediately."""
return self._lock.blocking
@property
def loop(self) -> asyncio.AbstractEventLoop | None:
"""The event loop used for ``run_in_executor``, or ``None`` for the running loop."""
return self._loop
@property
def executor(self) -> futures.Executor | None:
"""The executor used for ``run_in_executor``, or ``None`` for the default executor."""
return self._executor
async def acquire_read(
self, timeout: float | None = None, *, blocking: bool | None = None
) -> AsyncAcquireSoftReadWriteReturnProxy:
"""
Acquire a shared read lock.
See :meth:`SoftReadWriteLock.acquire_read` for the full reentrancy / upgrade / fork semantics. The blocking
work runs inside ``run_in_executor`` so other coroutines on the same loop continue to progress while this
call waits.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:returns: a proxy usable as an async context manager to release the lock
:raises RuntimeError: if a write lock is already held, if this instance was invalidated by
:func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self._run(self._lock.acquire_read, timeout, blocking=blocking)
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
async def acquire_write(
self, timeout: float | None = None, *, blocking: bool | None = None
) -> AsyncAcquireSoftReadWriteReturnProxy:
"""
Acquire an exclusive write lock.
See :meth:`SoftReadWriteLock.acquire_write` for the two-phase writer-preferring semantics. The blocking
work runs inside ``run_in_executor``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:returns: a proxy usable as an async context manager to release the lock
:raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if
this instance was invalidated by :func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self._run(self._lock.acquire_write, timeout, blocking=blocking)
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
async def release(self, *, force: bool = False) -> None:
"""
Release one level of the current lock.
:param force: if ``True``, release the lock completely regardless of the current lock level
:raises RuntimeError: if no lock is currently held and *force* is ``False``
"""
await self._run(self._lock.release, force=force)
@asynccontextmanager
async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases a shared read lock.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a write lock is already held on this instance
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
@asynccontextmanager
async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases an exclusive write lock.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
await self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
async def close(self) -> None:
"""Release any held lock and release the underlying filesystem resources. Idempotent."""
await self._run(self._lock.close)
async def _run(self, func: Callable[..., object], *args: object, **kwargs: object) -> object:
loop = self._loop or asyncio.get_running_loop()
return await loop.run_in_executor(self._executor, functools.partial(func, *args, **kwargs))
__all__ = [
"AsyncAcquireSoftReadWriteReturnProxy",
"AsyncSoftReadWriteLock",
]
@@ -0,0 +1,935 @@
"""Cross-process and cross-host reader/writer lock built on :class:`SoftFileLock` primitives."""
from __future__ import annotations
import atexit
import hmac
import os
import re
import secrets
import socket
import stat
import sys
import threading
import time
import uuid
from contextlib import contextmanager, suppress
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Literal
from weakref import WeakValueDictionary
from filelock._api import AcquireReturnProxy
from filelock._error import Timeout
from filelock._soft import SoftFileLock
from filelock._util import ensure_directory_exists
if TYPE_CHECKING:
from collections.abc import Callable, Generator
_Mode = Literal["read", "write"]
_BREAK_SUFFIX = ".break"
_MAX_MARKER_SIZE = 1024
_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
_O_NONBLOCK = getattr(os, "O_NONBLOCK", 0)
# Retargeting os.utime to an open fd lets the heartbeat refresh the exact inode it just verified instead of
# re-resolving the path; Windows read-only handles cannot set times that way, so keep it Unix-only.
_SUPPORTS_UTIME_FD = sys.platform != "win32" and os.utime in os.supports_fd
# os.utime follows symlinks unless told not to; not every platform can refuse the follow, so probe support.
_SUPPORTS_UTIME_NOFOLLOW = os.utime in os.supports_follow_symlinks
# dirfd-relative I/O is a Unix-only optimization; Windows cannot ``os.open()`` a directory at all, and
# its ``os`` module skips dir_fd support entirely. When disabled, callers fall back to full-path ops.
_SUPPORTS_DIR_FD = sys.platform != "win32" and os.open in os.supports_dir_fd
_all_instances: WeakValueDictionary[Path, SoftReadWriteLock] = WeakValueDictionary()
_all_instances_lock = threading.Lock()
_atexit_registered = False
_fork_registered = False
@dataclass(frozen=True)
class _Paths:
state: str
write: str
readers: str
@dataclass
class _Locks:
internal: threading.Lock
transaction: threading.Lock
state: SoftFileLock
@dataclass(frozen=True)
class _MarkerInfo:
token: str
pid: int
hostname: str
@dataclass
class _Hold:
"""Everything that exists only while a lock is held; ``None`` when the instance has no lock."""
level: int
mode: _Mode
write_thread_id: int | None
marker_name: str
is_reader: bool
token: str
heartbeat_thread: _HeartbeatThread
heartbeat_stop: threading.Event
class _SoftRWMeta(type):
_instances: WeakValueDictionary[Path, SoftReadWriteLock]
_instances_lock: threading.Lock
def __call__( # noqa: PLR0913
cls,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
heartbeat_interval: float = 30.0,
stale_threshold: float | None = None,
poll_interval: float = 0.25,
) -> SoftReadWriteLock:
if not is_singleton:
return super().__call__(
lock_file,
timeout,
blocking=blocking,
is_singleton=is_singleton,
heartbeat_interval=heartbeat_interval,
stale_threshold=stale_threshold,
poll_interval=poll_interval,
)
normalized = Path(lock_file).resolve()
with cls._instances_lock:
instance = cls._instances.get(normalized)
if instance is None:
instance = super().__call__(
lock_file,
timeout,
blocking=blocking,
is_singleton=is_singleton,
heartbeat_interval=heartbeat_interval,
stale_threshold=stale_threshold,
poll_interval=poll_interval,
)
cls._instances[normalized] = instance
elif instance.timeout != timeout or instance.blocking != blocking:
msg = (
f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking},"
f" cannot be changed to timeout={timeout}, blocking={blocking}"
)
raise ValueError(msg)
return instance
class SoftReadWriteLock(metaclass=_SoftRWMeta):
"""
Cross-process and cross-host reader/writer lock built on :class:`SoftFileLock` primitives.
Use this class instead of :class:`~filelock.ReadWriteLock` when the lock file lives on a network
filesystem (NFS, Lustre with ``-o flock``, HPC cluster shared storage). ``ReadWriteLock`` is backed
by SQLite and cannot run on NFS because SQLite's ``fcntl`` locking is unreliable there.
Layout on disk for a lock at ``foo.lock``:
- ``foo.lock.state`` — a :class:`SoftFileLock` taken only during state transitions (microseconds).
- ``foo.lock.write`` — writer marker; its presence means a writer is claiming or holding the lock.
- ``foo.lock.readers/<host>.<pid>.<uuid>`` — one file per reader.
Each marker stores a random token (``secrets.token_hex(16)``), the holder's pid, and the holder's
hostname. A daemon heartbeat thread refreshes ``mtime`` on every held marker. A marker whose mtime
has not advanced in ``stale_threshold`` seconds may be evicted by any process on any host, giving
correct behavior when a compute node crashes with a lock held.
Writer acquire is two-phase and writer-preferring: phase 1 claims ``.write`` (blocking any new
reader), phase 2 waits for existing readers to drain. Writer starvation is impossible.
Reentrancy, upgrade/downgrade rules, thread pinning, and singleton caching by resolved path match
:class:`~filelock.ReadWriteLock`.
Forking while holding a lock invalidates the inherited instance in the child so the child cannot
double-own the lock with its parent; ``release()`` on a fork-invalidated instance is a no-op, and
the child must re-acquire if it needs a lock.
Trust boundary: protects against same-UID non-cooperating processes (one host or cross-host) and
same-host different-UID users via ``0o600`` / ``0o700`` permissions. Does not protect against root
compromise, NTP tampering on same-UID cross-host nodes, or multi-tenant mounts where hostile
co-tenants share the UID.
:param lock_file: path to the lock file; sidecar state/write/readers live next to it
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
:param is_singleton: if ``True``, reuse existing instances for the same resolved path
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
:param stale_threshold: seconds of ``mtime`` inactivity before a marker is stale; defaults to
``3 * heartbeat_interval``, matching etcd's ``LeaseKeepAlive`` convention
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
.. versionadded:: 3.27.0
"""
_instances: WeakValueDictionary[Path, SoftReadWriteLock] = WeakValueDictionary()
_instances_lock = threading.Lock()
def __init__( # noqa: PLR0913
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True, # noqa: ARG002
heartbeat_interval: float = 30.0,
stale_threshold: float | None = None,
poll_interval: float = 0.25,
) -> None:
if heartbeat_interval <= 0:
msg = f"heartbeat_interval must be positive, got {heartbeat_interval}"
raise ValueError(msg)
if stale_threshold is None:
stale_threshold = heartbeat_interval * 3
if stale_threshold <= heartbeat_interval:
msg = f"stale_threshold must exceed heartbeat_interval ({stale_threshold} <= {heartbeat_interval})"
raise ValueError(msg)
if poll_interval <= 0:
msg = f"poll_interval must be positive, got {poll_interval}"
raise ValueError(msg)
self.lock_file: str = os.fspath(lock_file)
self.timeout: float = timeout
self.blocking: bool = blocking
self.heartbeat_interval: float = heartbeat_interval
self.stale_threshold: float = stale_threshold
self.poll_interval: float = poll_interval
self._paths = _Paths(
state=f"{self.lock_file}.state",
write=f"{self.lock_file}.write",
readers=f"{self.lock_file}.readers",
)
ensure_directory_exists(self.lock_file)
self._locks = _Locks(
internal=threading.Lock(),
transaction=threading.Lock(),
state=SoftFileLock(self._paths.state, timeout=-1),
)
self._readers_dir_fd: int | None = None
self._hold: _Hold | None = None
self._fork_invalidated: bool = False
self._closed: bool = False
with _all_instances_lock:
_all_instances[Path(self.lock_file).resolve()] = self
_register_hooks()
@contextmanager
def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases a shared read lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a write lock is already held on this instance
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
self.release()
@contextmanager
def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases an exclusive write lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
self.release()
def acquire_read(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
"""
Acquire a shared read lock.
If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a
read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed). On the 0→1
transition a daemon heartbeat thread is started that refreshes the reader marker's ``mtime`` every
``heartbeat_interval`` seconds so peers on other hosts do not evict the marker as stale.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
``None`` uses the instance default
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a write lock is already held on this instance, if this instance was invalidated by
:func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
return self._acquire("read", timeout, blocking=blocking)
def acquire_write(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
"""
Acquire an exclusive write lock.
If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant).
Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not
allowed). Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises
:class:`RuntimeError`.
Writer acquisition runs in two phases. Phase 1 atomically claims ``<path>.write`` via ``O_CREAT | O_EXCL``,
which immediately blocks any new reader on any host. Phase 2 waits for existing readers to drain. Writer
starvation is impossible: new readers see ``<path>.write`` during phase 2 and wait behind the pending writer.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
``None`` uses the instance default
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if this
instance was invalidated by :func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
"""
return self._acquire("write", timeout, blocking=blocking)
def close(self) -> None:
"""
Release any held lock and release internal filesystem resources.
Idempotent. After calling this method the instance can no longer acquire locks — subsequent acquires raise
:class:`RuntimeError`. A fork-invalidated instance is closed without raising.
"""
self.release(force=True)
with self._locks.internal:
if self._closed:
return
self._closed = True
if self._readers_dir_fd is not None:
with suppress(OSError):
os.close(self._readers_dir_fd)
self._readers_dir_fd = None
def release(self, *, force: bool = False) -> None:
"""
Release one level of the current lock.
When the lock level reaches zero the heartbeat thread is stopped and the held marker file is unlinked. On a
fork-invalidated instance (that is, the child of a :func:`os.fork` call made while the parent held a lock)
this method is a no-op so inherited ``with`` blocks can unwind cleanly in the child.
:param force: if ``True``, release the lock completely regardless of the current lock level
:raises RuntimeError: if no lock is currently held and *force* is ``False``
"""
with self._locks.internal:
if self._fork_invalidated:
# Inherited state from the parent is meaningless in the child; clear any counters and return.
self._hold = None
return
hold = self._hold
if hold is None:
if force:
return
msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held"
raise RuntimeError(msg)
if force:
hold.level = 0
else:
hold.level -= 1
if hold.level > 0:
return
self._hold = None
# Order matters: signal → join → unlink. A late tick on a deleted marker is harmless, and the
# token check in the heartbeat callback would catch any re-acquisition race, but joining first
# removes even that theoretical race.
hold.heartbeat_stop.set()
hold.heartbeat_thread.join(timeout=self.heartbeat_interval + 1.0)
if hold.is_reader:
_unlink(hold.marker_name, dir_fd=self._readers_dir_fd)
else:
self._unlink_writer_marker_if_ours(hold.token)
def _unlink_writer_marker_if_ours(self, token: str) -> None:
# Remove the writer marker only while it still carries our token. If this holder was paused long
# enough (a stop-the-world GC pause, SIGSTOP, a suspended VM) for a peer to evict the marker as
# stale and claim the writer slot itself, the file now at <path>.write is the peer's live marker;
# unlinking it by path would let a second writer through and break mutual exclusion. The state lock
# serializes this against a concurrent break/claim, and the heartbeat is already stopped, so the
# token we read is authoritative. Mirrors the token re-check the stale-break path already does.
with self._locks.state:
read = _read_marker(self._paths.write)
if read is None:
return
info, _ = read
if info is None or not hmac.compare_digest(info.token, token):
return
_unlink(self._paths.write)
def _claim_writer_marker(self, token: str) -> bool:
# Claim the writer slot for ``token``. Must be called holding ``self._locks.state``. Evicts a
# stale marker first, then refuses to claim while a live ``.write`` exists so a peer holding the
# slot is waited out instead of overwritten.
_break_stale_marker(self._paths.write, stale_threshold=self.stale_threshold, now=time.time())
if _file_exists(self._paths.write):
return False
try:
_atomic_create_marker(self._paths.write, token)
except FileExistsError:
return False
return True
def _touch_writer_marker_if_ours(self, token: str) -> bool:
# Refresh the writer marker through a single O_NOFOLLOW fd, but only while it still carries our
# token. Returns False when the marker is gone or now belongs to a peer that reclaimed the slot,
# so the caller can re-claim rather than keep a stranger's marker alive. Mirrors _refresh_marker.
fd = _open_marker(self._paths.write)
if fd is None:
return False
try:
try:
data = os.read(fd, _MAX_MARKER_SIZE + 1)
except OSError: # pragma: no cover - e.g. EAGAIN from a hostile FIFO that has a writer attached
return False
info = _parse_marker_bytes(data)
if info is None or not hmac.compare_digest(info.token, token):
return False
with suppress(OSError):
_touch(self._paths.write, fd=fd)
return True
finally:
os.close(fd)
@classmethod
def get_lock(
cls,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
) -> SoftReadWriteLock:
"""
Return the singleton :class:`SoftReadWriteLock` for *lock_file*.
:param lock_file: path to the lock file; sidecar state/write/readers live next to it
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: the singleton lock instance
:raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values
"""
return cls(lock_file, timeout, blocking=blocking)
def _acquire(
self,
mode: _Mode,
timeout: float | None,
*,
blocking: bool | None,
) -> AcquireReturnProxy:
timeout = self.timeout if timeout is None else timeout
blocking = self.blocking if blocking is None else blocking
with self._locks.internal:
if self._fork_invalidated:
msg = f"SoftReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance"
raise RuntimeError(msg)
if self._closed:
msg = f"SoftReadWriteLock on {self.lock_file} has been closed"
raise RuntimeError(msg)
if self._hold is not None:
return self._validate_reentrant(mode)
start = time.perf_counter()
if not blocking:
acquired = self._locks.transaction.acquire(blocking=False)
elif timeout == -1:
acquired = self._locks.transaction.acquire(blocking=True)
else:
acquired = self._locks.transaction.acquire(blocking=True, timeout=timeout)
if not acquired:
raise Timeout(self.lock_file) from None
try:
return self._do_acquire_inner(mode, timeout, start, blocking=blocking)
finally:
self._locks.transaction.release()
def _do_acquire_inner(
self,
mode: _Mode,
effective_timeout: float,
start: float,
*,
blocking: bool,
) -> AcquireReturnProxy:
with self._locks.internal:
if self._hold is not None:
return self._validate_reentrant(mode)
deadline = None if effective_timeout == -1 else start + effective_timeout
token = secrets.token_hex(16)
if mode == "write":
marker_name, is_reader = self._acquire_writer_slot(token, deadline=deadline, blocking=blocking)
else:
marker_name, is_reader = self._acquire_reader_slot(token, deadline=deadline, blocking=blocking)
stop_event = threading.Event()
heartbeat = _HeartbeatThread(
refresh=self._refresh_marker,
interval=self.heartbeat_interval,
stop_event=stop_event,
name=f"filelock-heartbeat-{id(self):x}",
)
with self._locks.internal:
self._hold = _Hold(
level=1,
mode=mode,
write_thread_id=threading.get_ident() if mode == "write" else None,
marker_name=marker_name,
is_reader=is_reader,
token=token,
heartbeat_thread=heartbeat,
heartbeat_stop=stop_event,
)
heartbeat.start()
return AcquireReturnProxy(lock=self)
def _validate_reentrant(self, mode: _Mode) -> AcquireReturnProxy:
hold = self._hold
assert hold is not None # noqa: S101
if hold.mode != mode:
opposite = "write" if mode == "read" else "read"
direction = "downgrade" if mode == "read" else "upgrade"
msg = (
f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): "
f"already holding a {opposite} lock ({direction} not allowed)"
)
raise RuntimeError(msg)
if mode == "write" and (cur := threading.get_ident()) != hold.write_thread_id:
msg = (
f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) "
f"from thread {cur} while it is held by thread {hold.write_thread_id}"
)
raise RuntimeError(msg)
hold.level += 1
return AcquireReturnProxy(lock=self)
def _acquire_writer_slot(
self,
token: str,
*,
deadline: float | None,
blocking: bool,
) -> tuple[str, bool]:
# Phase 2 scans readers/ via dirfd (where supported), so we need it open even though writers never
# create files inside.
self._open_readers_dir()
def try_claim_writer() -> bool:
with self._locks.state:
return self._claim_writer_marker(token)
def readers_drained_touching() -> bool:
with self._locks.state:
# Refresh our writer marker every scan iteration so phase 2 does not exceed
# ``stale_threshold`` under contention and get evicted. The refresh only happens while
# the marker is still ours: if we were paused past ``stale_threshold`` a peer can evict
# the stale marker and reclaim ``.write`` with its own token, and touching that path
# blindly would keep the peer's live marker alive and let this acquire finish as though
# we still held the slot, admitting a second writer. When the claim is no longer ours we
# re-claim the slot here (waiting behind the peer if it currently holds it) rather than
# trusting the foreign marker, mirroring the token re-check the stale-break and release
# paths already rely on.
if not self._touch_writer_marker_if_ours(token) and not self._claim_writer_marker(token):
return False
self._break_stale_readers(time.time())
return not self._any_readers()
self._wait_for(try_claim_writer, deadline=deadline, blocking=blocking)
try:
self._wait_for(readers_drained_touching, deadline=deadline, blocking=blocking)
except Timeout:
# Give up our writer claim so readers can make progress again, but only while the marker is
# still ours: a peer may have evicted it as stale and claimed the slot while phase 2 waited.
self._unlink_writer_marker_if_ours(token)
raise
return self._paths.write, False
def _acquire_reader_slot(
self,
token: str,
*,
deadline: float | None,
blocking: bool,
) -> tuple[str, bool]:
self._open_readers_dir()
reader_name = f"{uuid.uuid4().hex}.{os.getpid()}"
dir_fd = self._readers_dir_fd
full_reader_path = str(Path(self._paths.readers) / reader_name)
def try_claim_reader() -> bool:
with self._locks.state:
_break_stale_marker(self._paths.write, stale_threshold=self.stale_threshold, now=time.time())
if _file_exists(self._paths.write):
return False
if dir_fd is not None:
_atomic_create_marker(reader_name, token, dir_fd=dir_fd)
else: # pragma: win32 cover
_atomic_create_marker(full_reader_path, token)
return True
self._wait_for(try_claim_reader, deadline=deadline, blocking=blocking)
return (reader_name if dir_fd is not None else full_reader_path), True
def _wait_for(
self,
predicate: Callable[[], bool],
*,
deadline: float | None,
blocking: bool,
) -> None:
while True:
if predicate():
return
now = time.perf_counter()
if not blocking:
raise Timeout(self.lock_file)
if deadline is not None and now >= deadline:
raise Timeout(self.lock_file)
sleep_for = self.poll_interval
if deadline is not None:
sleep_for = min(sleep_for, max(deadline - now, 0.0))
time.sleep(sleep_for)
def _open_readers_dir(self) -> None:
readers_path = Path(self._paths.readers)
with suppress(FileExistsError):
readers_path.mkdir(mode=0o700)
# mkdir has no O_NOFOLLOW, so verify via lstat that we did not land on an attacker-placed symlink
# or a regular file before we open or scan inside.
st = os.lstat(self._paths.readers)
if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode):
msg = f"{self._paths.readers} exists but is not a directory or is a symlink; refusing to use it"
raise RuntimeError(msg)
if self._readers_dir_fd is None and _SUPPORTS_DIR_FD:
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | _O_NOFOLLOW
self._readers_dir_fd = os.open(self._paths.readers, flags)
def _any_readers(self) -> bool:
for _ in self._iter_reader_entries():
return True
return False
def _iter_reader_entries(self) -> Generator[tuple[str, bool]]:
"""
Yield ``(name, dirfd_relative)`` pairs for every live reader marker.
``dirfd_relative`` is ``True`` when *name* should be passed to ``dir_fd=``-aware syscalls; ``False``
when *name* is a full path because dirfd-relative I/O is unavailable on this platform.
"""
if self._readers_dir_fd is not None:
with os.scandir(self._readers_dir_fd) as it:
for entry in it:
if not _is_housekeeping_name(entry.name):
yield entry.name, True
return
readers_path = Path(self._paths.readers) # pragma: win32 cover
with os.scandir(readers_path) as it: # pragma: win32 cover
for entry in it: # pragma: win32 cover
if not _is_housekeeping_name(entry.name): # pragma: win32 cover
yield str(readers_path / entry.name), False # pragma: win32 cover
def _break_stale_readers(self, now: float) -> None:
names: list[tuple[str, int | None]] = []
try:
for name, dirfd_relative in self._iter_reader_entries():
names.append((name, self._readers_dir_fd if dirfd_relative else None))
except OSError: # pragma: no cover - transient NFS scandir hiccup
return
for name, fd in names:
_break_stale_marker(name, stale_threshold=self.stale_threshold, now=now, dir_fd=fd)
def _refresh_marker(self) -> bool:
with self._locks.internal:
hold = self._hold
if hold is None: # pragma: no cover - race between stop_event.set and join
return False
marker_name = hold.marker_name
token = hold.token
dir_fd = self._readers_dir_fd if hold.is_reader else None
# Open once with O_NOFOLLOW and touch that exact descriptor. Doing the refresh through the verified fd
# (instead of re-opening by name) closes the window where a peer unlinks our marker and drops a symlink
# or a different file at the path between the read and the touch: utime then lands on the inode we
# verified, or nowhere. A failed open means the marker is gone or was swapped out -- a peer evicted us
# -- so stop the heartbeat at once instead of waiting a tick.
fd = _open_marker(marker_name, dir_fd=dir_fd)
if fd is None:
return False
try:
try:
data = os.read(fd, _MAX_MARKER_SIZE + 1)
except OSError: # pragma: no cover - e.g. EAGAIN from a hostile FIFO that has a writer attached
return False
info = _parse_marker_bytes(data)
# Token mismatch means another process already evicted our marker and created its own; stop the
# thread so it does not keep a stranger's file alive.
if info is None or not hmac.compare_digest(info.token, token):
return False
# A transient touch failure (ESTALE / EIO on the NFS-style filesystems this lock targets) must not
# kill the heartbeat thread: the read above just confirmed the marker is still ours, so swallow the
# error and retry on the next tick rather than letting the lease lapse while we still hold the lock.
with suppress(OSError):
_touch(marker_name, fd=fd)
return True
finally:
os.close(fd)
def _reset_after_fork_in_child(self) -> None: # pragma: no cover - fork child not tracked
# Replace every lock this instance owns with a fresh one; the inherited locks may still be held
# by threads that no longer exist in the child. The readers dirfd and the SoftFileLock state
# mutex both get dropped for the same reason — the child re-creates them on its next acquire.
self._locks = _Locks(
internal=threading.Lock(),
transaction=threading.Lock(),
state=SoftFileLock(self._paths.state, timeout=-1),
)
self._hold = None
self._readers_dir_fd = None
self._fork_invalidated = True
class _HeartbeatThread(threading.Thread):
def __init__(
self,
refresh: Callable[[], bool],
interval: float,
stop_event: threading.Event,
name: str,
) -> None:
super().__init__(name=name, daemon=True)
self._refresh = refresh
self._interval = interval
self._stop_event = stop_event
def run(self) -> None:
while not self._stop_event.wait(self._interval):
if not self._refresh():
self._stop_event.set()
return
def _atomic_create_marker(name: str, token: str, *, dir_fd: int | None = None) -> None:
# O_NOFOLLOW blocks the symlink-overwrite attack where an attacker pre-creates the marker path as a
# symlink pointing at a victim file. Mode 0o600 keeps the token unreadable to other users.
flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | _O_NOFOLLOW
if _SUPPORTS_DIR_FD and dir_fd is not None:
fd = os.open(name, flags, 0o600, dir_fd=dir_fd)
else:
fd = os.open(name, flags, 0o600)
try:
content = f"{token}\n{os.getpid()}\n{socket.gethostname()}\n".encode("ascii")
os.write(fd, content)
finally:
os.close(fd)
def _open_marker(name: str, *, dir_fd: int | None = None) -> int | None:
# The file is ours; these guard a hostile mid-flight swap. O_NOFOLLOW rejects a symlink; O_NONBLOCK keeps
# a real FIFO from blocking the open forever, so it reads as a malformed marker instead of wedging a peer
# that holds the state lock.
flags = os.O_RDONLY | _O_NOFOLLOW | _O_NONBLOCK
try:
return os.open(name, flags, dir_fd=dir_fd) if _SUPPORTS_DIR_FD and dir_fd is not None else os.open(name, flags)
except OSError:
return None
def _read_marker(name: str, *, dir_fd: int | None = None) -> tuple[_MarkerInfo | None, float] | None:
fd = _open_marker(name, dir_fd=dir_fd)
if fd is None:
return None
try:
st = os.fstat(fd)
# A legitimate marker is always a regular file, so anything else at the path -- above all a FIFO -- is
# reported as a malformed marker (its mtime still drives stale eviction) without being read. Reading is
# where platforms diverge: an empty non-blocking read yields 0 bytes on Linux/macOS but EAGAIN on
# FreeBSD, and the EAGAIN used to abort the stale-break and wedge the acquire until timeout (#587).
if not stat.S_ISREG(st.st_mode):
return None, st.st_mtime
data = os.read(fd, _MAX_MARKER_SIZE + 1)
except OSError: # pragma: no cover - marker vanished or turned unreadable between open and read
return None
finally:
os.close(fd)
return _parse_marker_bytes(data), st.st_mtime
def _parse_marker_bytes(data: bytes) -> _MarkerInfo | None:
# Trust nothing about attacker-controlled markers; any deviation returns None so callers fall through
# to stale cleanup. ``re.match`` caches compiled patterns internally, so the regex is built only once
# despite being defined inline.
if not data or len(data) > _MAX_MARKER_SIZE:
return None
try:
text = data.decode("ascii")
except UnicodeDecodeError:
return None
match = re.match(
r"""
\A # start of string
(?P<token> [0-9a-f]{32} ) \n # 128-bit hex token
(?P<pid> [1-9][0-9]{0,9} ) \n # decimal pid: no leading zero, ≤ 10 digits
(?P<hostname> [\x21-\x7e]{1,253}) # printable non-whitespace ASCII (RFC 1123 hostname limit)
\n* # tolerate sloppy writers that append extra newlines
\Z # end of string
""",
text,
re.VERBOSE,
)
if match is None:
return None
pid = int(match["pid"], 10)
if pid > 2**31 - 1:
return None
return _MarkerInfo(token=match["token"], pid=pid, hostname=match["hostname"])
def _break_stale_marker( # noqa: PLR0911
name: str,
*,
stale_threshold: float,
now: float,
dir_fd: int | None = None,
) -> bool:
# Atomic break pattern: read → rename to unique break-name → re-verify → unlink. The rename gives us a
# private name nobody else can touch; if the re-verify sees a newer mtime or a different token, the
# legitimate holder's heartbeat fired between read and rename and we must abort (leaving the .break.*
# file behind rather than rollback-renaming, because rollback is itself racy).
read_result = _read_marker(name, dir_fd=dir_fd)
if read_result is None:
return False
info_before, mtime_before = read_result
if now - mtime_before <= stale_threshold:
return False
if info_before is None:
_unlink(name, dir_fd=dir_fd)
return True
break_name = f"{name}{_BREAK_SUFFIX}.{os.getpid()}.{secrets.token_hex(16)}"
try:
if _SUPPORTS_DIR_FD and dir_fd is not None:
os.rename(name, break_name, src_dir_fd=dir_fd, dst_dir_fd=dir_fd)
else:
Path(name).rename(break_name)
except OSError: # pragma: no cover - race where the marker vanishes between read and rename
return False
read_after = _read_marker(break_name, dir_fd=dir_fd)
if read_after is None: # pragma: no cover - race where a peer unlinks the break-name file
return False
info_after, mtime_after = read_after
if info_after is None: # pragma: no cover - content replaced post-rename by a racing peer
_unlink(break_name, dir_fd=dir_fd)
return True
if not hmac.compare_digest(info_before.token, info_after.token): # pragma: no cover - race only
return False
if mtime_after > mtime_before: # pragma: no cover - heartbeat raced our rename
return False
_unlink(break_name, dir_fd=dir_fd)
return True
def _unlink(name: str, *, dir_fd: int | None = None) -> None:
with suppress(FileNotFoundError):
if _SUPPORTS_DIR_FD and dir_fd is not None:
# Path.unlink has no dir_fd support, so we stay on os.unlink for the dirfd path.
os.unlink(name, dir_fd=dir_fd)
else:
Path(name).unlink()
def _touch(name: str, *, fd: int | None = None) -> None:
# Prefer the already-open, already-verified fd so a peer that swaps a symlink or a different file in at the
# path after our O_NOFOLLOW read cannot redirect the touch: utime then targets the inode behind the fd.
# Where the platform cannot utime an fd, fall back to a path-based touch that still refuses to follow a
# symlink where supported -- matching the O_NOFOLLOW reads used everywhere else here.
if fd is not None and _SUPPORTS_UTIME_FD:
os.utime(fd, None)
return
os.utime(name, None, follow_symlinks=not _SUPPORTS_UTIME_NOFOLLOW)
def _file_exists(path: str) -> bool:
try:
st = os.lstat(path)
except FileNotFoundError:
return False
return stat.S_ISREG(st.st_mode)
def _is_housekeeping_name(name: str) -> bool:
return name.startswith(".") or _BREAK_SUFFIX in name
def _reset_all_after_fork() -> None: # pragma: no cover - fork child, not tracked by coverage
global _all_instances_lock # noqa: PLW0603
# User-created threading locks do not auto-reset across fork: any lock held by a parent thread stays
# locked in the child with no owner to release it. Replace the module-level lock and every instance's
# locks with fresh ones; the child is single-threaded at this point so no synchronization is needed.
_all_instances_lock = threading.Lock()
for instance in list(_all_instances.values()):
instance._reset_after_fork_in_child() # noqa: SLF001
def _cleanup_all_instances() -> None: # pragma: no cover - runs from atexit at interpreter shutdown
for instance in list(_all_instances.values()):
with suppress(Exception):
instance.release(force=True)
def _register_hooks() -> None:
global _atexit_registered, _fork_registered # noqa: PLW0603
if not _atexit_registered:
atexit.register(_cleanup_all_instances)
_atexit_registered = True
# after_in_child replaces inherited state so the child cannot double-own any lock the parent held.
if not _fork_registered and hasattr(os, "register_at_fork"):
os.register_at_fork(after_in_child=_reset_all_after_fork)
_fork_registered = True
__all__ = [
"SoftReadWriteLock",
]
@@ -0,0 +1,114 @@
from __future__ import annotations
import os
import sys
import warnings
from contextlib import suppress
from errno import EAGAIN, ENOSYS, EWOULDBLOCK
from pathlib import Path
from typing import cast
from ._api import BaseFileLock
from ._util import ensure_directory_exists
#: a flag to indicate if the fcntl API is available
has_fcntl = False
if sys.platform == "win32": # pragma: win32 cover
class UnixFileLock(BaseFileLock):
"""Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems."""
def _acquire(self) -> None:
raise NotImplementedError
def _release(self) -> None:
raise NotImplementedError
else: # pragma: win32 no cover
try:
import fcntl
_ = (fcntl.flock, fcntl.LOCK_EX, fcntl.LOCK_NB, fcntl.LOCK_UN)
except (ImportError, AttributeError):
pass
else:
has_fcntl = True
class UnixFileLock(BaseFileLock):
"""
Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.
The lock file is intentionally left in place after release. Unlinking a locked file on Unix
can split waiters across different inodes and break mutual exclusion for processes that
coordinate via the same path.
"""
def _acquire(self) -> None: # noqa: C901, PLR0912
ensure_directory_exists(self.lock_file)
open_flags = os.O_RDWR | os.O_TRUNC
o_nofollow = getattr(os, "O_NOFOLLOW", None)
if o_nofollow is not None:
open_flags |= o_nofollow
open_flags |= os.O_CREAT
open_mode = self._open_mode()
try:
fd = os.open(self.lock_file, open_flags, open_mode)
except FileNotFoundError:
# On FUSE/NFS, os.open(O_CREAT) is not atomic: LOOKUP + CREATE can be split, allowing a concurrent
# unlink() to delete the file between them. For valid paths, treat ENOENT as transient contention.
# For invalid paths (e.g., empty string), re-raise to avoid infinite retry loops.
if self.lock_file and Path(self.lock_file).parent.exists():
return
raise
except PermissionError:
# Sticky-bit dirs (e.g. /tmp): O_CREAT fails if the file is owned by another user (#317).
# Fall back to opening the existing file without O_CREAT.
if not Path(self.lock_file).exists():
raise
try:
fd = os.open(self.lock_file, open_flags & ~os.O_CREAT, open_mode)
except FileNotFoundError:
return
if self.has_explicit_mode:
with suppress(PermissionError):
os.fchmod(fd, self._context.mode)
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as exception:
os.close(fd)
if exception.errno == ENOSYS:
with suppress(OSError):
Path(self.lock_file).unlink()
self._fallback_to_soft_lock()
self._acquire()
return
if exception.errno not in {EAGAIN, EWOULDBLOCK}:
raise
else:
# The file may have been unlinked by a concurrent _release() between our open() and flock().
# A lock on an unlinked inode is useless — discard and let the retry loop start fresh.
if os.fstat(fd).st_nlink == 0:
os.close(fd)
else:
self._context.lock_file_fd = fd
def _fallback_to_soft_lock(self) -> None:
from ._soft import SoftFileLock # noqa: PLC0415
warnings.warn("flock not supported on this filesystem, falling back to SoftFileLock", stacklevel=2)
from .asyncio import AsyncSoftFileLock, BaseAsyncFileLock # noqa: PLC0415
self.__class__ = AsyncSoftFileLock if isinstance(self, BaseAsyncFileLock) else SoftFileLock
def _release(self) -> None:
fd = cast("int", self._context.lock_file_fd)
self._context.lock_file_fd = None
fcntl.flock(fd, fcntl.LOCK_UN)
with suppress(OSError): # close can raise EIO on FUSE/Docker bind-mount filesystems
os.close(fd)
__all__ = [
"UnixFileLock",
"has_fcntl",
]
@@ -0,0 +1,95 @@
from __future__ import annotations
import os
import secrets
import stat
import sys
from errno import EACCES, EISDIR
from pathlib import Path
def raise_on_not_writable_file(filename: str) -> None:
"""
Raise an exception if attempting to open the file for writing would fail.
Separates files that can never be written from files that are writable but currently locked.
:param filename: file to check
:raises OSError: as if the file was opened for writing.
"""
try:
# lstat, not stat: it settles exists-and-writable in one syscall, and a hostile symlink planted at the lock
# path would otherwise make this inspect the link target, letting an attacker turn a contended acquire into a
# misleading PermissionError / IsADirectoryError and probe that target's attributes. The real open passes
# O_NOFOLLOW and refuses the symlink anyway.
file_stat = os.lstat(filename)
except OSError:
return # does not exist, or an error the caller cannot act on
# No mtime guard: the old `if st_mtime != 0` skip existed for NFS/Linux quirks where os.lstat could return an
# all-zero struct, which it no longer does. Skipping on mtime 0 let a read-only file or a directory at the lock
# path pass as missing, so acquire() blocked forever on an open that cannot succeed.
if not (file_stat.st_mode & stat.S_IWUSR):
raise PermissionError(EACCES, "Permission denied", filename)
if stat.S_ISDIR(file_stat.st_mode):
if sys.platform == "win32": # pragma: win32 cover
raise PermissionError(EACCES, "Permission denied", filename)
raise IsADirectoryError(EISDIR, "Is a directory", filename) # pragma: win32 no cover
def ensure_directory_exists(filename: Path | str) -> None:
"""
Ensure the directory containing the file exists (create it if necessary).
:param filename: file.
"""
Path(filename).parent.mkdir(parents=True, exist_ok=True)
def break_lock_file(lock_file: str, mtime_before: float, ino_before: int) -> None:
"""
Atomically break a stale lock file that was judged stale at modification time *mtime_before*.
The file is renamed to a process-private name before being unlinked, so two processes breaking the same lock
cannot delete each other's work (only one rename of a given inode succeeds; the loser gets ``OSError``). After the
rename the file is re-checked: a newer modification time, or a different inode than *ino_before*, means a peer
recreated the lock between the stale decision and the rename, so we grabbed a live file and must abort, leaving the
renamed file in place rather than rolling back (a rollback rename is itself racy — same trade-off as the soft
read/write marker break). The inode check matters because filesystems with coarse modification-time granularity
(NFS, FAT) can give a same-second recreation the old mtime, so mtime alone would not catch it and a live lock would
be unlinked; the inode is the reliable identity, mirroring the token re-check in the soft read/write marker break.
``lstat`` is used so a hostile symlink swapped in after the decision is not followed.
The break name carries a random token so it is unguessable and unique per attempt. Without it two breakers in the
same process share ``<lock>.break.<pid>``, and a second break can rename a freshly recreated live lock onto that
path in the window between the re-verify ``lstat`` above and the ``unlink`` below, so we would delete a live lock
the inode check just approved. A private name means nobody else can target our break path, matching the soft
read/write marker break.
:param lock_file: path to the lock file to break.
:param mtime_before: modification time observed when the lock was judged stale.
:param ino_before: inode number observed when the lock was judged stale.
:raises OSError: if the rename fails (e.g. the file vanished or is not owned in a sticky directory).
"""
break_path = f"{lock_file}.break.{os.getpid()}.{secrets.token_hex(16)}"
Path(lock_file).rename(break_path)
try:
st_after = os.lstat(break_path)
except OSError:
return
if st_after.st_mtime > mtime_before or st_after.st_ino != ino_before:
return
Path(break_path).unlink()
__all__ = [
"break_lock_file",
"ensure_directory_exists",
"raise_on_not_writable_file",
]
@@ -0,0 +1,111 @@
from __future__ import annotations
import os
import sys
from contextlib import suppress
from errno import EACCES
from pathlib import Path
from typing import cast
from ._api import BaseFileLock
from ._util import ensure_directory_exists, raise_on_not_writable_file
if sys.platform == "win32": # pragma: win32 cover
import ctypes
import msvcrt
from ctypes import wintypes
# Windows API constants for reparse point detection
FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400
INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
# Load kernel32.dll
_kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
_kernel32.GetFileAttributesW.argtypes = [wintypes.LPCWSTR]
_kernel32.GetFileAttributesW.restype = wintypes.DWORD
def _is_reparse_point(path: str) -> bool:
"""
Check if a path is a reparse point (symlink, junction, etc.) on Windows.
:param path: Path to check
:returns: True if path is a reparse point, False otherwise
:raises OSError: If GetFileAttributesW fails for reasons other than file-not-found
"""
attrs = _kernel32.GetFileAttributesW(path)
if attrs == INVALID_FILE_ATTRIBUTES:
# File doesn't exist yet - that's fine, we'll create it
err = ctypes.get_last_error()
if err == 2: # noqa: PLR2004 # ERROR_FILE_NOT_FOUND
return False
if err == 3: # noqa: PLR2004 # ERROR_PATH_NOT_FOUND
return False
# Some other error - let caller handle it
return False
return bool(attrs & FILE_ATTRIBUTE_REPARSE_POINT)
class WindowsFileLock(BaseFileLock):
"""
Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems.
Lock file cleanup: Windows attempts to delete the lock file after release, but deletion is
not guaranteed in multi-threaded scenarios where another thread holds an open handle. The lock
file may persist on disk, which does not affect lock correctness.
"""
def _acquire(self) -> None:
raise_on_not_writable_file(self.lock_file)
ensure_directory_exists(self.lock_file)
# Security check: Refuse to open reparse points (symlinks, junctions)
# This prevents TOCTOU symlink attacks (CVE-TBD)
if _is_reparse_point(self.lock_file):
msg = f"Lock file is a reparse point (symlink/junction): {self.lock_file}"
raise OSError(msg)
flags = (
os.O_RDWR # open for read and write
| os.O_CREAT # create file if not exists
)
try:
fd = os.open(self.lock_file, flags, self._open_mode())
except OSError as exception:
if exception.errno != EACCES: # has no access to this lock
raise
else:
try:
msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
except OSError as exception:
os.close(fd) # close file first
if exception.errno != EACCES: # file is already locked
raise
else:
self._context.lock_file_fd = fd
def _release(self) -> None:
fd = cast("int", self._context.lock_file_fd)
self._context.lock_file_fd = None
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
os.close(fd)
with suppress(OSError):
Path(self.lock_file).unlink()
else: # pragma: win32 no cover
class WindowsFileLock(BaseFileLock):
"""Uses the :func:`msvcrt.locking` function to hard lock the lock file on Windows systems."""
def _acquire(self) -> None:
raise NotImplementedError
def _release(self) -> None:
raise NotImplementedError
__all__ = [
"WindowsFileLock",
]
@@ -0,0 +1,414 @@
"""An asyncio-based implementation of the file lock."""
from __future__ import annotations
import asyncio
import contextlib
import logging
import os
import time
from dataclasses import dataclass
from inspect import iscoroutinefunction
from threading import local
from typing import TYPE_CHECKING, Any, NoReturn, TypeVar
from ._api import _UNSET_FILE_MODE, BaseFileLock, FileLockContext, FileLockMeta, _canonical
from ._error import Timeout
from ._soft import SoftFileLock
from ._unix import UnixFileLock
from ._windows import WindowsFileLock
if TYPE_CHECKING:
import sys
from collections.abc import Callable
from concurrent import futures
from types import TracebackType
if sys.version_info >= (3, 11): # pragma: no cover (py311+)
from typing import Self
else: # pragma: no cover (<py311)
from typing_extensions import Self
_LOGGER = logging.getLogger("filelock")
@dataclass
class AsyncFileLockContext(FileLockContext):
"""A dataclass which holds the context for a ``BaseAsyncFileLock`` object."""
#: Whether run in executor
run_in_executor: bool = True
#: The executor
executor: futures.Executor | None = None
#: The loop
loop: asyncio.AbstractEventLoop | None = None
class AsyncThreadLocalFileContext(AsyncFileLockContext, local):
"""A thread local version of the ``FileLockContext`` class."""
class AsyncAcquireReturnProxy:
"""A context-aware object that will release the lock file when exiting."""
def __init__(self, lock: BaseAsyncFileLock) -> None: # noqa: D107
self.lock = lock
async def __aenter__(self) -> BaseAsyncFileLock: # noqa: D105
return self.lock
async def __aexit__( # noqa: D105
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
await self.lock.release()
_AT = TypeVar("_AT", bound="BaseAsyncFileLock")
class AsyncFileLockMeta(FileLockMeta):
def __call__( # ty: ignore[invalid-method-override] # noqa: PLR0913
cls: type[_AT], # noqa: N805
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = False, # noqa: FBT001, FBT002
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
loop: asyncio.AbstractEventLoop | None = None,
run_in_executor: bool = True,
executor: futures.Executor | None = None,
) -> _AT:
if thread_local and run_in_executor:
msg = "run_in_executor is not supported when thread_local is True"
raise ValueError(msg)
return super().__call__(
lock_file=lock_file,
timeout=timeout,
mode=mode,
thread_local=thread_local,
blocking=blocking,
is_singleton=is_singleton,
poll_interval=poll_interval,
lifetime=lifetime,
loop=loop,
run_in_executor=run_in_executor,
executor=executor,
)
class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
"""
Base class for asynchronous file locks.
.. versionadded:: 3.15.0
"""
_deadlock_holder_desc: str = "BaseAsyncFileLock instance in this task"
def __init__( # noqa: PLR0913
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = False, # noqa: FBT001, FBT002
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
loop: asyncio.AbstractEventLoop | None = None,
run_in_executor: bool = True,
executor: futures.Executor | None = None,
) -> None:
"""
Create a new lock object.
:param lock_file: path to the file
:param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the
acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a
negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock.
:param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and
default ACLs, preserving POSIX default ACL inheritance in shared directories.
:param thread_local: Whether this object's internal context should be thread local or not. If this is set to
``False`` then the lock will be reentrant across threads. When ``True`` (the default), **all fields of the
lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``,
``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change
the value seen by another thread; threads that did not perform the write continue to see the value supplied
at construction time. If you need configuration values to be visible across threads, construct the lock
with ``thread_local=False``.
:param blocking: whether the lock should be blocking or not
:param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock
file. This is useful if you want to use the lock object for reentrant locking without needing to pass the
same object around.
:param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value
in the acquire method, if no poll_interval value (``None``) is given.
:param lifetime: maximum time in seconds a lock can be held before it is considered expired. When set, a waiting
process will break a lock whose file modification time is older than ``lifetime`` seconds. ``None`` (the
default) means locks never expire.
:param loop: The event loop to use. If not specified, the running event loop will be used.
:param run_in_executor: If this is set to ``True`` then the lock will be acquired in an executor.
:param executor: The executor to use. If not specified, the default executor will be used.
"""
self._is_thread_local = thread_local
self._is_singleton = is_singleton
# Create the context. Note that external code should not work with the context directly and should instead use
# properties of this class.
kwargs: dict[str, Any] = {
"lock_file": os.fspath(lock_file),
"timeout": timeout,
"mode": mode,
"blocking": blocking,
"poll_interval": poll_interval,
"lifetime": lifetime,
"loop": loop,
"run_in_executor": run_in_executor,
"executor": executor,
}
self._context: AsyncFileLockContext = (AsyncThreadLocalFileContext if thread_local else AsyncFileLockContext)(
**kwargs
)
@property
def run_in_executor(self) -> bool:
"""Whether run in executor."""
return self._context.run_in_executor
@property
def executor(self) -> futures.Executor | None:
"""The executor."""
return self._context.executor
@executor.setter
def executor(self, value: futures.Executor | None) -> None: # pragma: no cover
"""
Change the executor.
:param futures.Executor | None value: the new executor or ``None``
"""
self._context.executor = value
@property
def loop(self) -> asyncio.AbstractEventLoop | None:
"""The event loop."""
return self._context.loop
async def acquire( # ty: ignore[invalid-method-override]
self,
timeout: float | None = None,
poll_interval: float | None = None,
*,
blocking: bool | None = None,
cancel_check: Callable[[], bool] | None = None,
) -> AsyncAcquireReturnProxy:
"""
Try to acquire the file lock.
:param timeout: maximum wait time for acquiring the lock, ``None`` means use the default
:attr:`~BaseFileLock.timeout` is and if ``timeout < 0``, there is no timeout and this method will block
until the lock could be acquired
:param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default
:attr:`~BaseFileLock.poll_interval`
:param blocking: defaults to True. If False, function will return immediately if it cannot obtain a lock on the
first attempt. Otherwise, this method will block until the timeout expires or the lock is acquired.
:param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll
iteration. When triggered, raises :class:`~Timeout` just like an expired timeout.
:returns: a context object that will unlock the file when the context is exited
:raises Timeout: if fails to acquire lock within the timeout period
.. code-block:: python
# You can use this method in the context manager (recommended)
with lock.acquire():
pass
# Or use an equivalent try-finally construct:
lock.acquire()
try:
pass
finally:
lock.release()
"""
# Use the default timeout, if no timeout is provided.
if timeout is None:
timeout = self._context.timeout
if blocking is None:
blocking = self._context.blocking
if poll_interval is None:
poll_interval = self._context.poll_interval
# Increment the number right at the beginning. We can still undo it, if something fails.
self._context.lock_counter += 1
canonical = _canonical(self.lock_file)
self._raise_if_would_deadlock(canonical, timeout=timeout, blocking=blocking)
start_time = time.perf_counter()
try:
await self._async_poll_until_acquired(
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
poll_interval=poll_interval,
start_time=start_time,
)
except BaseException: # Something did go wrong, so decrement the counter.
self._undo_acquire(canonical)
raise
self._commit_acquire(canonical)
return AsyncAcquireReturnProxy(lock=self)
async def _async_poll_until_acquired(
self,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
timeout: float,
poll_interval: float,
start_time: float,
) -> None:
lock_id = id(self)
lock_filename = self.lock_file
while True:
if not self.is_locked:
self._try_break_expired_lock()
_LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename)
await self._run_internal_method(self._acquire)
if self.is_locked:
_LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename)
return
if self._check_give_up(
lock_id,
lock_filename,
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
start_time=start_time,
):
raise Timeout(lock_filename)
msg = "Lock %s not acquired on %s, waiting %s seconds ..."
_LOGGER.debug(msg, lock_id, lock_filename, poll_interval)
await asyncio.sleep(poll_interval)
async def release(self, force: bool = False) -> None: # ty: ignore[invalid-method-override] # noqa: FBT001, FBT002
"""
Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file
itself may be deleted automatically, the behavior is platform-specific.
:param force: If true, the lock counter is ignored and the lock is released in every case.
"""
if self.is_locked:
self._context.lock_counter -= 1
if self._context.lock_counter == 0 or force:
lock_id, lock_filename = id(self), self.lock_file
_LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename)
await self._run_internal_method(self._release)
self._context.lock_counter = 0
self._drop_registry_entry()
_LOGGER.debug("Lock %s released on %s", lock_id, lock_filename)
async def _run_internal_method(self, method: Callable[[], Any]) -> None:
if iscoroutinefunction(method):
await method()
elif self.run_in_executor:
await asyncio.get_running_loop().run_in_executor(self.executor, method)
else:
method()
def __enter__(self) -> NoReturn:
"""Sync context manager entry is not supported because lock acquisition is a coroutine."""
msg = "Use `async with` — acquire/release are coroutines and cannot be awaited in a sync context manager."
raise NotImplementedError(msg)
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: object,
) -> None:
"""Sync context manager exit is not supported because lock release is a coroutine."""
msg = "Use `async with` — acquire/release are coroutines and cannot be awaited in a sync context manager."
raise NotImplementedError(msg)
async def __aenter__(self) -> Self:
"""
Acquire the lock.
:returns: the lock object
"""
await self.acquire()
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
"""
Release the lock.
:param exc_type: the exception type if raised
:param exc_value: the exception value if raised
:param traceback: the exception traceback if raised
"""
await self.release()
def __del__(self) -> None:
"""Release on deletion — safe to call during GC even when no event loop is running."""
with contextlib.suppress(Exception):
try:
loop = asyncio.get_running_loop()
except RuntimeError:
# No running loop — try stored loop or create one
loop = self._context.loop if self._context.loop and not self._context.loop.is_closed() else None
if loop is None:
return
if not loop.is_running(): # pragma: no cover
loop.run_until_complete(self.release(force=True))
else:
loop.create_task(self.release(force=True))
class AsyncSoftFileLock(SoftFileLock, BaseAsyncFileLock):
"""Simply watches the existence of the lock file."""
class AsyncUnixFileLock(UnixFileLock, BaseAsyncFileLock):
"""Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems."""
class AsyncWindowsFileLock(WindowsFileLock, BaseAsyncFileLock):
"""Uses the :func:`msvcrt.locking` to hard lock the lock file on windows systems."""
__all__ = [
"AsyncAcquireReturnProxy",
"AsyncSoftFileLock",
"AsyncUnixFileLock",
"AsyncWindowsFileLock",
"BaseAsyncFileLock",
]
@@ -0,0 +1,24 @@
# file generated by vcs-versioning
# don't change, don't track in version control
from __future__ import annotations
__all__ = [
"__version__",
"__version_tuple__",
"version",
"version_tuple",
"__commit_id__",
"commit_id",
]
version: str
__version__: str
__version_tuple__: tuple[int | str, ...]
version_tuple: tuple[int | str, ...]
commit_id: str | None
__commit_id__: str | None
__version__ = version = '3.29.7'
__version_tuple__ = version_tuple = (3, 29, 7)
__commit_id__ = commit_id = None
@@ -0,0 +1 @@
pip
@@ -0,0 +1,182 @@
Metadata-Version: 2.4
Name: juc500-xfer
Version: 0.1.0
Summary: Apple Silicon Mac ↔ Windows 11 file transfer over j5create JUC500
Requires-Python: >=3.10
Description-Content-Type: text/markdown
Requires-Dist: pyusb==1.2.1
# JUC500 File Transfer (Apple Silicon ↔ Windows 11)
j5create **JUC500** USB 3.0 Wormhole 케이블로 **Apple Silicon Mac**과 **Windows 11** 사이 파일을 직접 전송하는 오픈 구현입니다.
## 왜 공식 앱이 Silicon Mac에서 안 되나
실측/분석 결과:
| 항목 | 내용 |
|------|------|
| USB ID | `0711:7500` / Product: **Smart Data Link** |
| 가상 CD | `WORMHOLE` (FAT12, 공식 설치본 포함) |
| 번들/공식 Mac 앱 (2024, v1.0.1463.47) | **x86_64 전용**, Apple Silicon 네이티브 **arm64 없음** |
| USB 스택 | deprecated `IOUSBDevice` API (`kIOUSBDeviceInterfaceID500`) |
| 핵심 라이브러리 | KaiJet / OTi `OTiTransfer.framework` — bounding·alive·XML UPipe |
Intel Mac에서는 Rosetta 없이(구버전) 또는 Rosetta로 동작할 수 있으나, Apple Silicon + 최신 macOS에서는 공식 Wormhole 경로가 깨집니다. 이 프로젝트는 **libusb로 Vendor/CDC 인터페이스를 직접 열고**, 자체 프레임 프로토콜로 파일을 보냅니다.
공식 Windows Wormhole과 호환되지 않습니다. **양쪽 모두 이 프로그램을 실행**해야 합니다.
## 장치 인터페이스 (요약)
```
IF0 CDC Comm INT 0x81
IF1 CDC Data BULK 0x02 / 0x83
IF2 Mass Storage WORMHOLE CD (OS 점유 — 건드리지 않음)
IF3 HID mouse (KM — OS 점유)
IF4 HID keyboard (KM — OS 점유)
IF5 Vendor BULK 0x08/0x89 (데이터) + 0x0A/0x8B (keepalive)
```
## 요구 사항
### macOS (Apple Silicon)
- Python 3.10+
- libusb: `brew install libusb`
- (선택) 공식 Wormhole 앱이 떠 있으면 종료
```bash
cd /path/to/juc500
python3 -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
python -m juc500_xfer info
```
### Windows 11
1. Python 3.10+ 설치
2. [libusb](https://libusb.info/) 또는 이 저장소 `vendor/libusb-1.0.dll` 을 PATH/`vendor/` 에 배치
3. **Zadig**로 `Smart Data Link` 복합 장치의
- **Interface 5** (Vendor Specific)
- **Interface 1** (CDC Data, 권장)
에 **WinUSB** 드라이버 설치
4. 공식 j5create Wormhole / 자동실행 소프트웨어는 **종료·제거** (같은 인터페이스를 점유함)
```powershell
cd juc500
py -3 -m venv .venv
.\.venv\Scripts\Activate.ps1
# vendor\wheels\pyusb-*.whl 사용 (PyPI 불필요)
python -m pip install --no-index --find-links=vendor\wheels -r requirements.txt
pip install -e .
python -m juc500_xfer info
```
또는 `scripts\setup_windows.bat` 실행.
**장치가 안 보이면:**
```powershell
# libusb DLL 복사 (필수)
copy vendor\libusb-1.0.dll .venv\Scripts\libusb-1.0.dll
python -m juc500_xfer doctor
```
Zadig로 **Interface 5 (MI_05)** 에 WinUSB 설치 여부를 확인하세요.
케이블 가상 CD의 공식 **Wormhole**가 IF5를 점유하면 Access denied가 납니다.
GUI **Wormhole 종료** 버튼 또는 `python -m juc500_xfer kill-wormhole` 로 종료할 수 있습니다 (연결·실행 시에도 자동 종료).
## 사용법
### GUI (권장)
**macOS**
```bash
# 최초 1회
./scripts/setup_macos.sh
# 이후
./start.sh
```
**Windows 10/11** (Git Bash)
```bash
# 최초 1회 (PowerShell/CMD)
scripts\setup_windows.bat
# 이후 (Git Bash)
./start.sh
```
인자를 넘기면 CLI로 동작합니다: `./start.sh doctor`, `./start.sh kill-wormhole` 등.
양쪽 PC에 케이블을 꽂고, **먼저 수신 쪽**, 이어서 송신 쪽을 실행합니다.
**Windows (수신 예)**
```bash
./start.sh recv --dest "$USERPROFILE/Downloads" -y
```
**Mac (송신 예)**
```bash
./start.sh send ~/Desktop/archive.zip
```
## Windows 설치파일
산출물:
| 파일 | 용도 |
|------|------|
| `JUC500-Setup-win64.exe` | Inno Setup 설치 프로그램 |
| `JUC500-portable.exe` | 단일 실행 포터블 EXE |
| `JUC500-win64-portable.zip` | 폴더형 포터블 (+ `setup.bat`) |
**Mac(Apple Silicon)에서 빌드:**
```bash
./build-installer-windows.sh --ci
# → dist-windows/ 에 위 파일 다운로드 (GitHub Actions)
```
**Windows PC에서 빌드:**
```bat
build-installer-windows.bat
```
(Inno Setup 6 필요 시 Setup.exe 생성. 없어도 portable exe/zip 은 생성됩니다.)
사용 전 Win11에서 Zadig WinUSB 설정이 필요합니다 (`docs/WINDOWS.md`).
## GUI
fileShare와 동일한 **듀얼 패널(이 PC | 전송 | 상대 PC)** 웹 UI입니다.
```bash
python -m juc500_xfer gui
# → http://127.0.0.1:8765/
```
양쪽 PC에서 GUI를 실행한 뒤 **연결**을 누르세요.
**입력공유(가장자리 제어권):** 양쪽 모두 동일 소스 + `pynput` 설치 후 「입력공유 켜기」.
상대 PC를 오른쪽에 둔다고 가정합니다. 마우스 오른쪽 끝 → 상대 제어, 왼쪽 끝(또는 Ctrl+Shift+←) → 이 PC 복귀.
macOS는 **손쉬운 사용** 권한이 필요합니다.
Windows 설치본은 `JUC500.exe` / `JUC500-portable.exe` 더블클릭으로 동일 GUI가 열립니다.
## 프로토콜
Vendor BULK(`0x08`/`0x89`) 위에 길이·CRC 프레임(`J5FX` 매직)을 올리고 `HELLO` 핸드셰이크로 연결을 만듭니다. 보조 파이프(`0x0A`/`0x8B`)로 keepalive를내어 링크를 유지합니다. 파일은 SHA-256으로 무결성을 검증합니다.
공식 OTi XML / bounding 프로토콜은 재구현하지 않았습니다.
## 분석 메모
자세한 USB·바이너리 분석은 [`docs/ANALYSIS.md`](docs/ANALYSIS.md) 를 참고하세요.
@@ -0,0 +1,12 @@
../../Scripts/juc500.exe,sha256=g9LL2SAOaQkiofqJtNJE_zzBS477mSXFCPDb1rwChrY,108351
__editable__.juc500_xfer-0.1.0.pth,sha256=Ev-t039m5yovBpOx1Evlo6vzg6w4eUdbJhko3aw88u4,93
__editable___juc500_xfer_0_1_0_finder.py,sha256=A-DUae7cdvVm2j81qu_zz9afN46Kap4nNTCiIU946hE,3465
__pycache__/__editable___juc500_xfer_0_1_0_finder.cpython-311.pyc,,
juc500_xfer-0.1.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
juc500_xfer-0.1.0.dist-info/METADATA,sha256=3jpRWwfpBQNds_5Sp3whz3TSQ9B5_QKadSBXvZyH_Z4,5786
juc500_xfer-0.1.0.dist-info/RECORD,,
juc500_xfer-0.1.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
juc500_xfer-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
juc500_xfer-0.1.0.dist-info/direct_url.json,sha256=LGQCxhElUDMVi1Kp9jwBp3Pi5mKZd5MZW9yh8w0xAtI,71
juc500_xfer-0.1.0.dist-info/entry_points.txt,sha256=oRp3XTmaPD3kyKVSu7XpamFa-gYkpoDoYO3420k3c0g,48
juc500_xfer-0.1.0.dist-info/top_level.txt,sha256=AzVqKZOjpSLOvEtPEIkVxKRXE5nR1037fXWGbtryAko,12
@@ -0,0 +1,5 @@
Wheel-Version: 1.0
Generator: setuptools (83.0.0)
Root-Is-Purelib: true
Tag: py3-none-any
@@ -0,0 +1 @@
{"dir_info": {"editable": true}, "url": "file:///C:/dev/juc500/juc500"}
@@ -0,0 +1,2 @@
[console_scripts]
juc500 = juc500_xfer.cli:main
@@ -0,0 +1 @@
juc500_xfer
@@ -0,0 +1 @@
pip
@@ -0,0 +1,112 @@
Metadata-Version: 2.4
Name: packaging
Version: 26.2
Summary: Core utilities for Python packages
Author-email: Donald Stufft <donald@stufft.io>
Requires-Python: >=3.8
Description-Content-Type: text/x-rst
License-Expression: Apache-2.0 OR BSD-2-Clause
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: 3.14
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Programming Language :: Python :: Free Threading :: 4 - Resilient
Classifier: Typing :: Typed
License-File: LICENSE
License-File: LICENSE.APACHE
License-File: LICENSE.BSD
Project-URL: Documentation, https://packaging.pypa.io/
Project-URL: Source, https://github.com/pypa/packaging
packaging
=========
.. start-intro
Reusable core utilities for various Python Packaging
`interoperability specifications <https://packaging.python.org/specifications/>`_.
This library provides utilities that implement the interoperability
specifications which have clearly one correct behaviour (eg: :pep:`440`)
or benefit greatly from having a single shared implementation (eg: :pep:`425`).
.. end-intro
The ``packaging`` project includes the following: version handling, specifiers,
markers, requirements, tags, metadata, lockfiles, utilities.
Documentation
-------------
The `documentation`_ provides information and the API for the following:
- Version Handling
- Specifiers
- Markers
- Licenses
- Requirements
- Metadata
- Tags
- Lockfiles (pylock)
- Direct URL helpers
- Dependency groups
- Errors
- Utilities
Installation
------------
Use ``pip`` to install these utilities::
pip install packaging
The ``packaging`` library uses calendar-based versioning (``YY.N``).
Discussion
----------
If you run into bugs, you can file them in our `issue tracker`_.
You can also join discussions on `GitHub Discussions`_ to ask questions or get involved.
.. _`documentation`: https://packaging.pypa.io/
.. _`issue tracker`: https://github.com/pypa/packaging/issues
.. _`GitHub Discussions`: https://github.com/pypa/packaging/discussions
Code of Conduct
---------------
Everyone interacting in the packaging project's codebases, issue trackers, chat
rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.
.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md
Contributing
------------
The ``CONTRIBUTING.rst`` file outlines how to contribute to this project as
well as how to report a potential security issue. The documentation for this
project also covers information about `project development`_ and `security`_.
.. _`project development`: https://packaging.pypa.io/en/latest/development/
.. _`security`: https://packaging.pypa.io/en/latest/security/
Project History
---------------
Please review the ``CHANGELOG.rst`` file or the `Changelog documentation`_ for
recent changes and project history.
.. _`Changelog documentation`: https://packaging.pypa.io/en/latest/changelog/
@@ -0,0 +1,48 @@
packaging-26.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
packaging-26.2.dist-info/METADATA,sha256=T5y815M0FaR5P3dnyYoralEsgj_IHIczeBVwXyMOyr8,3543
packaging-26.2.dist-info/RECORD,,
packaging-26.2.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
packaging-26.2.dist-info/licenses/LICENSE,sha256=ytHvW9NA1z4HS6YU0m996spceUDD2MNIUuZcSQlobEg,197
packaging-26.2.dist-info/licenses/LICENSE.APACHE,sha256=DVQuDIgE45qn836wDaWnYhSdxoLXgpRRKH4RuTjpRZQ,10174
packaging-26.2.dist-info/licenses/LICENSE.BSD,sha256=tw5-m3QvHMb5SLNMFqo5_-zpQZY2S8iP8NIYDwAo-sU,1344
packaging/__init__.py,sha256=QhMEdPu2XogrJzV3S0KWS6t7l0I9k8EeDRJl4fnw87s,494
packaging/__pycache__/__init__.cpython-311.pyc,,
packaging/__pycache__/_elffile.cpython-311.pyc,,
packaging/__pycache__/_manylinux.cpython-311.pyc,,
packaging/__pycache__/_musllinux.cpython-311.pyc,,
packaging/__pycache__/_parser.cpython-311.pyc,,
packaging/__pycache__/_structures.cpython-311.pyc,,
packaging/__pycache__/_tokenizer.cpython-311.pyc,,
packaging/__pycache__/dependency_groups.cpython-311.pyc,,
packaging/__pycache__/direct_url.cpython-311.pyc,,
packaging/__pycache__/errors.cpython-311.pyc,,
packaging/__pycache__/markers.cpython-311.pyc,,
packaging/__pycache__/metadata.cpython-311.pyc,,
packaging/__pycache__/pylock.cpython-311.pyc,,
packaging/__pycache__/requirements.cpython-311.pyc,,
packaging/__pycache__/specifiers.cpython-311.pyc,,
packaging/__pycache__/tags.cpython-311.pyc,,
packaging/__pycache__/utils.cpython-311.pyc,,
packaging/__pycache__/version.cpython-311.pyc,,
packaging/_elffile.py,sha256=-sKkptYqzYw2-x3QByJa5mB4rfPWu1pxkZHRx1WAFCY,3211
packaging/_manylinux.py,sha256=Hf6nB0cOrayEs96-p3oIXAgGnFquv20DO5l-o2_Xnv0,9559
packaging/_musllinux.py,sha256=Z6swjH3MA7XS3qXnmMN7QPhqP3fnoYI0eQ18e9-HgAE,2707
packaging/_parser.py,sha256=Kf2nsDw4c54X82pY8ba4F02Bve6OygGMAjL-Begqcew,11698
packaging/_structures.py,sha256=60jRbF78p8z5MKnNd6cAprgOadCJHV0DlmUmRBqFZcs,1109
packaging/_tokenizer.py,sha256=tFU2Wr-ZZJdAbkXLEJo7qUQDJaIkfft9DqaifiEND7A,5391
packaging/dependency_groups.py,sha256=XZIAVFK9uHG4RCGprmJn3VInUWMesxha_kytJuMO9eY,10218
packaging/direct_url.py,sha256=eKmbDiPP1sLV4Mj_kCSZqqknrIyVO9Sr7JpF8KCjp4U,10917
packaging/errors.py,sha256=6hfEYXAf8v_IF65-lFadJOMIieBP2xIKtyEXjG1nGIs,2680
packaging/licenses/__init__.py,sha256=_Jx0XRiD_58palsWnyLrLuh59ZpGCPIPXLKdZo9OJvQ,7293
packaging/licenses/__pycache__/__init__.cpython-311.pyc,,
packaging/licenses/__pycache__/_spdx.cpython-311.pyc,,
packaging/licenses/_spdx.py,sha256=WW7DXiyg68up_YND_wpRYlr1SHhiV4FfJLQffghhMxQ,51122
packaging/markers.py,sha256=8fDIUhAF6YMnCNB5FSiwh9pEIusiFzAF73J-0OB8bTk,17055
packaging/metadata.py,sha256=crAh0E3GVGVqPlu6EdRFsaG-Y6UYznTUqjuGKRGPv6c,38770
packaging/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
packaging/pylock.py,sha256=G_1gncTmDbRLY1jo4VDI9Uw-b5IErh_Q9V_BbVJTmD8,33890
packaging/requirements.py,sha256=dd1c9aa1gp5NI6btF6UFRQjPn1nxQXnE_T34yDDTEpc,4383
packaging/specifiers.py,sha256=Mfp8avQg0lVot17to9lVKBtZD1FsWBTItoGwFUZ3wtg,71514
packaging/tags.py,sha256=NQ1weo69_Sjte3xBZ1I_G63CIgCmaN0C24mz-z3hGYo,34224
packaging/utils.py,sha256=M7-JMKic2sP1YtV_8aW7eVGB-x3ADuKCiSrsVeCd2Uo,9848
packaging/version.py,sha256=Y1aTtxe3sn2xOMa5BdI85-AcHuybbanOVkEvvSRRC8I,38369
@@ -0,0 +1,4 @@
Wheel-Version: 1.0
Generator: flit 3.12.0
Root-Is-Purelib: true
Tag: py3-none-any
@@ -0,0 +1,3 @@
This software is made available under the terms of *either* of the licenses
found in LICENSE.APACHE or LICENSE.BSD. Contributions to this software is made
under the terms of *both* these licenses.
@@ -0,0 +1,177 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
@@ -0,0 +1,23 @@
Copyright (c) Donald Stufft and individual contributors.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,15 @@
# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the BSD License. See the LICENSE file in the root of this repository
# for complete details.
__title__ = "packaging"
__summary__ = "Core utilities for Python packages"
__uri__ = "https://github.com/pypa/packaging"
__version__ = "26.2"
__author__ = "Donald Stufft and individual contributors"
__email__ = "donald@stufft.io"
__license__ = "BSD-2-Clause or Apache-2.0"
__copyright__ = f"2014 {__author__}"

Some files were not shown because too many files have changed in this diff Show More