From 80cc71edc172b395db8f14beb7add9a61c4cc2b6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2016 12:58:40 +0200 Subject: [PATCH 1/8] doc(README): add waffle.io info [skip ci] --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c999dcaae..f08d1b900 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ New BSD License. See the LICENSE file. [![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg)](https://travis-ci.org/gitpython-developers/GitPython) [![Code Climate](https://codeclimate.com/github/gitpython-developers/GitPython/badges/gpa.svg)](https://codeclimate.com/github/gitpython-developers/GitPython) [![Documentation Status](https://readthedocs.org/projects/gitpython/badge/?version=stable)](https://readthedocs.org/projects/gitpython/?badge=stable) +[![Stories in Ready](https://badge.waffle.io/gitpython-developers/GitPython.png?label=ready&title=Ready)](https://waffle.io/gitpython-developers/GitPython) +[![Throughput Graph](https://graphs.waffle.io/gitpython-developers/GitPython/throughput.svg)](https://waffle.io/gitpython-developers/GitPython/metrics/throughput) Now that there seems to be a massive user base, this should be motivation enough to let git-python return to a proper state, which means From 2d37049a815b11b594776d34be50e9c0ba8df497 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2016 15:17:54 +0200 Subject: [PATCH 2/8] doc(platforms): inform more clearly about best-effort This has been the case for Windows as well, and is now made official. Certain tests already fail on windows, for example. --- README.md | 5 +++++ doc/source/index.rst | 1 - doc/source/intro.rst | 8 +++++++- doc/source/whatsnew.rst | 25 ------------------------- 4 files changed, 12 insertions(+), 27 deletions(-) delete mode 100644 doc/source/whatsnew.rst diff --git a/README.md b/README.md index f08d1b900..b3308af2a 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,11 @@ codebase for `__del__` implementations and call these yourself when you see fit. Another way assure proper cleanup of resources is to factor out GitPython into a separate process which can be dropped periodically. +#### Best-effort for Python 2.6 and Windows support + +This means that support for these platforms is likely to worsen over time +as they are kept alive solely by their users, or not. + ### RUNNING TESTS *Important*: Right after cloning this repository, please be sure to have executed the `init-tests-after-clone.sh` script in the repository root. Otherwise you will encounter test failures. diff --git a/doc/source/index.rst b/doc/source/index.rst index 1079c5c76..69fb573a4 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -9,7 +9,6 @@ GitPython Documentation :maxdepth: 2 intro - whatsnew tutorial reference roadmap diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 1c1b0d1b4..1766f8ae0 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -15,7 +15,7 @@ Requirements * `Python`_ 2.7 or newer Since GitPython 2.0.0. Please note that python 2.6 is still reasonably well supported, but might - deteriorate over time. + deteriorate over time. Support is provided on a best-effort basis only. * `Git`_ 1.7.0 or newer It should also work with older versions, but it may be that some operations involving remotes will not work as expected. @@ -75,6 +75,12 @@ codebase for `__del__` implementations and call these yourself when you see fit. Another way assure proper cleanup of resources is to factor out GitPython into a separate process which can be dropped periodically. +Best-effort for Python 2.6 and Windows support +---------------------------------------------- + +This means that support for these platforms is likely to worsen over time +as they are kept alive solely by their users, or not. + Getting Started =============== diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst deleted file mode 100644 index e0d39b099..000000000 --- a/doc/source/whatsnew.rst +++ /dev/null @@ -1,25 +0,0 @@ - -################ -Whats New in 0.3 -################ -GitPython 0.3 is the first step in creating a hybrid which uses a pure python implementations for all simple git features which can be implemented without significant performance penalties. Everything else is still performed using the git command, which is nicely integrated and easy to use. - -Its biggest strength, being the support for all git features through the git command itself, is a weakness as well considering the possibly vast amount of times the git command is being started up. Depending on the actual command being performed, the git repository will be initialized on many of these invocations, causing additional overhead for possibly tiny operations. - -Keeping as many major operations in the python world will result in improved caching benefits as certain data structures just have to be initialized once and can be reused multiple times. This mode of operation may improve performance when altering the git database on a low level, and is clearly beneficial on operating systems where command invocations are very slow. - -**************** -Object Databases -**************** -An object database provides a simple interface to query object information or to write new object data. Objects are generally identified by their 20 byte binary sha1 value during query. - -GitPython uses the ``gitdb`` project to provide a pure-python implementation of the git database, which includes reading and writing loose objects, reading pack files and handling alternate repositories. - -The great thing about this is that ``Repo`` objects can use any object database, hence it easily supports different implementations with different performance characteristics. If you are thinking in extremes, you can implement your own database representation, which may be more efficient for what you want to do specifically, like handling big files more efficiently. - -************************ -Reduced Memory Footprint -************************ -Objects, such as commits, tags, trees and blobs now use 20 byte sha1 signatures internally, reducing their memory demands by 20 bytes per object, allowing you to keep more objects in memory at the same time. - -The internal caches of tree objects were improved to use less memory as well. From d6b1a9272455ef80f01a48ea22efc85b7f976503 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2016 17:10:38 +0200 Subject: [PATCH 3/8] fix(index): improve LockedFD handling Relying on the destructor will not work, even though the code used to rely on it. Now we handle failures more explicitly. Far from perfect, but a good start for a fix. Fixes #514 --- git/index/base.py | 14 ++++++++++++-- git/test/test_index.py | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/git/index/base.py b/git/index/base.py index 524b4568d..86eda41e6 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -118,13 +118,17 @@ def _set_cache_(self, attr): # read the current index # try memory map for speed lfd = LockedFD(self._file_path) + ok = False try: fd = lfd.open(write=False, stream=False) + ok = True except OSError: - lfd.rollback() # in new repositories, there may be no index, which means we are empty self.entries = dict() return + finally: + if not ok: + lfd.rollback() # END exception handling # Here it comes: on windows in python 2.5, memory maps aren't closed properly @@ -209,8 +213,14 @@ def write(self, file_path=None, ignore_extension_data=False): self.entries lfd = LockedFD(file_path or self._file_path) stream = lfd.open(write=True, stream=True) + ok = False - self._serialize(stream, ignore_extension_data) + try: + self._serialize(stream, ignore_extension_data) + ok = True + finally: + if not ok: + lfd.rollback() lfd.commit() diff --git a/git/test/test_index.py b/git/test/test_index.py index ca8778388..bce560891 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -135,6 +135,23 @@ def _cmp_tree_index(self, tree, index): raise AssertionError("CMP Failed: Missing entries in index: %s, missing in tree: %s" % (bset - iset, iset - bset)) # END assertion message + + @with_rw_repo('0.1.6') + def test_index_lock_handling(self, rw_repo): + def add_bad_blob(): + rw_repo.index.add([Blob(rw_repo, b'f' * 20, 'bad-permissions', 'foo')]) + + try: + ## 1st fail on purpose adding into index. + add_bad_blob() + except Exception as ex: + assert "cannot convert argument to integer" in str(ex) + + ## 2nd time should not fail due to stray lock file + try: + add_bad_blob() + except Exception as ex: + assert "index.lock' could not be obtained" not in str(ex) @with_rw_repo('0.1.6') def test_index_file_from_tree(self, rw_repo): From 0de60abc5eb71eff14faa0169331327141a5e855 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2016 17:17:27 +0200 Subject: [PATCH 4/8] fix(test): deal with py2 and py3 It ain't pretty, but should do the job. Related to #514 --- git/test/test_index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/git/test/test_index.py b/git/test/test_index.py index bce560891..178a59d2d 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -145,7 +145,9 @@ def add_bad_blob(): ## 1st fail on purpose adding into index. add_bad_blob() except Exception as ex: - assert "cannot convert argument to integer" in str(ex) + msg_py3 = "required argument is not an integer" + msg_py2 = "cannot convert argument to integer" + assert msg_py2 in str(ex) or msg_py3 in str(ex) ## 2nd time should not fail due to stray lock file try: From f73468bb9cb9e479a0b81e3766623c32802db579 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2016 17:20:28 +0200 Subject: [PATCH 5/8] fix(test): put `test_commits` back Thanks to @yarikoptic for catching this one ! --- git/test/test_repo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/git/test/test_repo.py b/git/test/test_repo.py index e24062c1b..d04a0f66f 100644 --- a/git/test/test_repo.py +++ b/git/test/test_repo.py @@ -115,6 +115,7 @@ def test_commit_from_revision(self): assert commit.type == 'commit' assert self.rorepo.commit(commit) == commit + def test_commits(self): mc = 10 commits = list(self.rorepo.iter_commits('0.1.6', max_count=mc)) assert len(commits) == mc From 7842e92ebaf3fc3380cc8d704afa3841f333748c Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Thu, 15 Sep 2016 00:59:36 +0200 Subject: [PATCH 6/8] test, deps: FIX `mock` deps on py3. + Del extra spaces, import os.path as osp --- git/cmd.py | 17 ++++++++--------- git/test/lib/asserts.py | 5 ++++- git/test/test_commit.py | 22 +++++++++++++--------- git/test/test_git.py | 6 +++++- setup.py | 4 +++- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/git/cmd.py b/git/cmd.py index ceea24425..1cc656bf5 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -5,7 +5,6 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php import os -import os.path import sys import select import logging @@ -213,11 +212,11 @@ def _deplete_buffer(fno, handler, buf_list, wg=None): def dashify(string): return string.replace('_', '-') - + def slots_to_dict(self, exclude=()): return dict((s, getattr(self, s)) for s in self.__slots__ if s not in exclude) - + def dict_to_slots_and__excluded_are_none(self, d, excluded=()): for k, v in d.items(): @@ -246,15 +245,15 @@ class Git(LazyMixin): """ __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version_info", "_git_options", "_environment") - + _excluded_ = ('cat_file_all', 'cat_file_header', '_version_info') - + def __getstate__(self): return slots_to_dict(self, exclude=self._excluded_) - + def __setstate__(self, d): dict_to_slots_and__excluded_are_none(self, d, excluded=self._excluded_) - + # CONFIGURATION # The size in bytes read from stdout when copying git's output to another stream max_chunk_size = 1024 * 64 @@ -267,7 +266,7 @@ def __setstate__(self, d): # value of Windows process creation flag taken from MSDN CREATE_NO_WINDOW = 0x08000000 - + # Provide the full path to the git executable. Otherwise it assumes git is in the path _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" GIT_PYTHON_GIT_EXECUTABLE = os.environ.get(_git_exec_env_var, git_exec_name) @@ -339,7 +338,7 @@ def wait(self, stderr=b''): if stderr is None: stderr = b'' stderr = force_bytes(stderr) - + status = self.proc.wait() def read_all_from_possibly_closed_stream(stream): diff --git a/git/test/lib/asserts.py b/git/test/lib/asserts.py index 60a888b3b..9edc49e08 100644 --- a/git/test/lib/asserts.py +++ b/git/test/lib/asserts.py @@ -16,7 +16,10 @@ assert_false ) -from mock import patch +try: + from unittest.mock import patch +except ImportError: + from mock import patch __all__ = ['assert_instance_of', 'assert_not_instance_of', 'assert_none', 'assert_not_none', diff --git a/git/test/test_commit.py b/git/test/test_commit.py index c05995033..805221ac1 100644 --- a/git/test/test_commit.py +++ b/git/test/test_commit.py @@ -34,7 +34,11 @@ import os from datetime import datetime from git.objects.util import tzoffset, utc -from mock import Mock + +try: + from unittest.mock import Mock +except ImportError: + from mock import Mock def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False): @@ -343,9 +347,9 @@ def test_gpgsig(self): cstream = BytesIO() cmt._serialize(cstream) assert re.search(r"^gpgsig $", cstream.getvalue().decode('ascii'), re.MULTILINE) - + self.assert_gpgsig_deserialization(cstream) - + cstream.seek(0) cmt.gpgsig = None cmt._deserialize(cstream) @@ -355,27 +359,27 @@ def test_gpgsig(self): cstream = BytesIO() cmt._serialize(cstream) assert not re.search(r"^gpgsig ", cstream.getvalue().decode('ascii'), re.MULTILINE) - + def assert_gpgsig_deserialization(self, cstream): assert 'gpgsig' in 'precondition: need gpgsig' - + class RepoMock: def __init__(self, bytestr): self.bytestr = bytestr - + @property def odb(self): class ODBMock: def __init__(self, bytestr): self.bytestr = bytestr - + def stream(self, *args): stream = Mock(spec_set=['read'], return_value=self.bytestr) stream.read.return_value = self.bytestr return ('binsha', 'typename', 'size', stream) - + return ODBMock(self.bytestr) - + repo_mock = RepoMock(cstream.getvalue()) for field in Commit.__slots__: c = Commit(repo_mock, b'x' * 20) diff --git a/git/test/test_git.py b/git/test/test_git.py index b46ac72d6..59796a3d0 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -6,7 +6,6 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php import os import sys -import mock import subprocess from git.test.lib import ( @@ -28,6 +27,11 @@ from git.compat import PY3 +try: + from unittest import mock +except ImportError: + import mock + class TestGit(TestBase): diff --git a/setup.py b/setup.py index 05c12b8f2..b3b43eb3b 100755 --- a/setup.py +++ b/setup.py @@ -68,8 +68,10 @@ def _stamp_version(filename): print("WARNING: Couldn't find version line in file %s" % filename, file=sys.stderr) install_requires = ['gitdb >= 0.6.4'] +test_requires = ['node'] if sys.version_info[:2] < (2, 7): install_requires.append('ordereddict') + test_requires.append('mock') # end setup( @@ -87,7 +89,7 @@ def _stamp_version(filename): license="BSD License", requires=['gitdb (>=0.6.4)'], install_requires=install_requires, - test_requirements=['mock', 'nose'] + install_requires, + test_requirements=test_requires + install_requires, zip_safe=False, long_description="""\ GitPython is a python library used to interact with Git repositories""", From 1210ec763e1935b95a3a909c61998fbd251b7575 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sun, 25 Sep 2016 12:02:52 +0200 Subject: [PATCH 7/8] apveyor: Wintest project with MINGW/Cygwin git (conda2.7&3.4/cpy-3.5) [travisci skip] --- .appveyor.yml | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 12 ++++----- 2 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..2af0ccdb5 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,74 @@ +# CI on Windows via appveyor +environment: + + matrix: + - PYTHON: "C:\\Miniconda" + PYTHON_VERSION: "2.7" + - PYTHON: "C:\\Miniconda" + PYTHON_VERSION: "2.7" + GIT_PATH: "C:\\cygwin64\\bin" + + - PYTHON: "C:\\Miniconda3-x64" + PYTHON_VERSION: "3.4" + - PYTHON: "C:\\Miniconda3-x64" + PYTHON_VERSION: "3.4" + GIT_PATH: "C:\\cygwin64\\bin" + + - PYTHON: "C:\Python35-x64" + PYTHON_VERSION: "3.5" + - PYTHON: "C:\Python35-x64" + PYTHON_VERSION: "3.5" + GIT_PATH: "C:\\cygwin64\\bin" + +install: + - set PATH=%PYTHON%;%PYTHON%\Scripts;%GIT_PATH%;%PATH% + + ## Print architecture, python & git used for debugging. + # + - | + uname -a + where git + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + conda info -a + + - conda install --yes --quiet pip + - pip install nose wheel coveralls + - IF "%PYTHON_VERSION%"=="2.7" ( + pip install mock + ) + + ## Copied from `init-tests-after-clone.sh`. + # + - | + git submodule update --init --recursive + git fetch --tags + git tag __testing_point__ + git checkout master || git checkout -b master + git reset --hard HEAD~1 + git reset --hard HEAD~1 + git reset --hard HEAD~1 + git reset --hard __testing_point__ + + ## For commits performed with the default user. + - | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + + - python setup.py develop + +build: off + +test_script: + - | + echo "+++ Checking archives for PyPI repo..." + python setup.py bdist_wheel + + - IF "%PYTHON_VERSION%"=="3.4" ( + nosetests -v --with-coverage + ) ELSE ( + nosetests -v + ) + +#on_success: +# - IF "%PYTHON_VERSION%"=="3.4" (coveralls) diff --git a/README.md b/README.md index b3308af2a..12159a06e 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Both commands will install the required package dependencies. A distribution package can be obtained for manual installation at: http://pypi.python.org/pypi/GitPython - + If you like to clone from source, you can do it like so: ```bash @@ -45,7 +45,7 @@ git submodule update --init --recursive #### Leakage of System Resources GitPython is not suited for long-running processes (like daemons) as it tends to -leak system resources. It was written in a time where destructors (as implemented +leak system resources. It was written in a time where destructors (as implemented in the `__del__` method) still ran deterministically. In case you still want to use it in such a context, you will want to search the @@ -61,7 +61,7 @@ as they are kept alive solely by their users, or not. ### RUNNING TESTS -*Important*: Right after cloning this repository, please be sure to have executed the `init-tests-after-clone.sh` script in the repository root. Otherwise you will encounter test failures. +*Important*: Right after cloning this repository, please be sure to have executed the `./init-tests-after-clone.sh` script in the repository root. Otherwise you will encounter test failures. The easiest way to run test is by using [tox](https://pypi.python.org/pypi/tox) a wrapper around virtualenv. It will take care of setting up environnements with the proper dependencies installed and execute test commands. To install it simply: @@ -70,8 +70,8 @@ The easiest way to run test is by using [tox](https://pypi.python.org/pypi/tox) Then run: tox - - + + For more fine-grained control, you can use `nose`. ### Contributions @@ -100,7 +100,7 @@ Please have a look at the [contributions file][contributing]. * Finally, set the upcoming version in the `VERSION` file, usually be incrementing the patch level, and possibly by appending `-dev`. Probably you want to `git push` once more. - + ### LICENSE New BSD License. See the LICENSE file. From 8fe4feee0fd7490c34c092c691a7c31446775a05 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sun, 25 Sep 2016 18:08:16 +0200 Subject: [PATCH 8/8] win: GC after every TC to fixes appveyor hang runs + Fixed the hangs at `test_git:TestGit.test_handle_process_output()`. [travisci skip] --- git/test/lib/helper.py | 2 ++ git/test/performance/test_commit.py | 4 ++++ git/test/test_docs.py | 7 ++++++- git/test/test_git.py | 4 ++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/git/test/lib/helper.py b/git/test/lib/helper.py index 8be2881c3..9488005f4 100644 --- a/git/test/lib/helper.py +++ b/git/test/lib/helper.py @@ -299,6 +299,8 @@ def setUpClass(cls): Dynamically add a read-only repository to our actual type. This way each test type has its own repository """ + import gc + gc.collect() cls.rorepo = Repo(GIT_REPO) @classmethod diff --git a/git/test/performance/test_commit.py b/git/test/performance/test_commit.py index b59c747ee..c60dc2fc4 100644 --- a/git/test/performance/test_commit.py +++ b/git/test/performance/test_commit.py @@ -17,6 +17,10 @@ class TestPerformance(TestBigRepoRW): + def tearDown(self): + import gc + gc.collect() + # ref with about 100 commits in its history ref_100 = '0.1.6' diff --git a/git/test/test_docs.py b/git/test/test_docs.py index b297363dc..2cd355b28 100644 --- a/git/test/test_docs.py +++ b/git/test/test_docs.py @@ -11,6 +11,11 @@ class Tutorials(TestBase): + + def tearDown(self): + import gc + gc.collect() + @with_rw_directory def test_init_repo_object(self, rw_dir): # [1-test_init_repo_object] @@ -64,7 +69,7 @@ def test_init_repo_object(self, rw_dir): assert repo.head.ref == repo.heads.master # head is a symbolic reference pointing to master assert repo.tags['0.3.5'] == repo.tag('refs/tags/0.3.5') # you can access tags in various ways too assert repo.refs.master == repo.heads['master'] # .refs provides access to all refs, i.e. heads ... - + if 'TRAVIS' not in os.environ: assert repo.refs['origin/master'] == repo.remotes.origin.refs.master # ... remotes ... assert repo.refs['0.3.5'] == repo.tags['0.3.5'] # ... and tags diff --git a/git/test/test_git.py b/git/test/test_git.py index 59796a3d0..534539d78 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -40,6 +40,10 @@ def setUpClass(cls): super(TestGit, cls).setUpClass() cls.git = Git(cls.rorepo.working_dir) + def tearDown(self): + import gc + gc.collect() + @patch.object(Git, 'execute') def test_call_process_calls_execute(self, git): git.return_value = ''