diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..2af0ccdb5 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,74 @@ +# CI on Windows via appveyor +environment: + + matrix: + - PYTHON: "C:\\Miniconda" + PYTHON_VERSION: "2.7" + - PYTHON: "C:\\Miniconda" + PYTHON_VERSION: "2.7" + GIT_PATH: "C:\\cygwin64\\bin" + + - PYTHON: "C:\\Miniconda3-x64" + PYTHON_VERSION: "3.4" + - PYTHON: "C:\\Miniconda3-x64" + PYTHON_VERSION: "3.4" + GIT_PATH: "C:\\cygwin64\\bin" + + - PYTHON: "C:\Python35-x64" + PYTHON_VERSION: "3.5" + - PYTHON: "C:\Python35-x64" + PYTHON_VERSION: "3.5" + GIT_PATH: "C:\\cygwin64\\bin" + +install: + - set PATH=%PYTHON%;%PYTHON%\Scripts;%GIT_PATH%;%PATH% + + ## Print architecture, python & git used for debugging. + # + - | + uname -a + where git + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + conda info -a + + - conda install --yes --quiet pip + - pip install nose wheel coveralls + - IF "%PYTHON_VERSION%"=="2.7" ( + pip install mock + ) + + ## Copied from `init-tests-after-clone.sh`. + # + - | + git submodule update --init --recursive + git fetch --tags + git tag __testing_point__ + git checkout master || git checkout -b master + git reset --hard HEAD~1 + git reset --hard HEAD~1 + git reset --hard HEAD~1 + git reset --hard __testing_point__ + + ## For commits performed with the default user. + - | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + + - python setup.py develop + +build: off + +test_script: + - | + echo "+++ Checking archives for PyPI repo..." + python setup.py bdist_wheel + + - IF "%PYTHON_VERSION%"=="3.4" ( + nosetests -v --with-coverage + ) ELSE ( + nosetests -v + ) + +#on_success: +# - IF "%PYTHON_VERSION%"=="3.4" (coveralls) diff --git a/README.md b/README.md index c999dcaae..12159a06e 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Both commands will install the required package dependencies. A distribution package can be obtained for manual installation at: http://pypi.python.org/pypi/GitPython - + If you like to clone from source, you can do it like so: ```bash @@ -45,7 +45,7 @@ git submodule update --init --recursive #### Leakage of System Resources GitPython is not suited for long-running processes (like daemons) as it tends to -leak system resources. It was written in a time where destructors (as implemented +leak system resources. It was written in a time where destructors (as implemented in the `__del__` method) still ran deterministically. In case you still want to use it in such a context, you will want to search the @@ -54,9 +54,14 @@ codebase for `__del__` implementations and call these yourself when you see fit. Another way assure proper cleanup of resources is to factor out GitPython into a separate process which can be dropped periodically. +#### Best-effort for Python 2.6 and Windows support + +This means that support for these platforms is likely to worsen over time +as they are kept alive solely by their users, or not. + ### RUNNING TESTS -*Important*: Right after cloning this repository, please be sure to have executed the `init-tests-after-clone.sh` script in the repository root. Otherwise you will encounter test failures. +*Important*: Right after cloning this repository, please be sure to have executed the `./init-tests-after-clone.sh` script in the repository root. Otherwise you will encounter test failures. The easiest way to run test is by using [tox](https://pypi.python.org/pypi/tox) a wrapper around virtualenv. It will take care of setting up environnements with the proper dependencies installed and execute test commands. To install it simply: @@ -65,8 +70,8 @@ The easiest way to run test is by using [tox](https://pypi.python.org/pypi/tox) Then run: tox - - + + For more fine-grained control, you can use `nose`. ### Contributions @@ -95,7 +100,7 @@ Please have a look at the [contributions file][contributing]. * Finally, set the upcoming version in the `VERSION` file, usually be incrementing the patch level, and possibly by appending `-dev`. Probably you want to `git push` once more. - + ### LICENSE New BSD License. See the LICENSE file. @@ -105,6 +110,8 @@ New BSD License. See the LICENSE file. [![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg)](https://travis-ci.org/gitpython-developers/GitPython) [![Code Climate](https://codeclimate.com/github/gitpython-developers/GitPython/badges/gpa.svg)](https://codeclimate.com/github/gitpython-developers/GitPython) [![Documentation Status](https://readthedocs.org/projects/gitpython/badge/?version=stable)](https://readthedocs.org/projects/gitpython/?badge=stable) +[![Stories in Ready](https://badge.waffle.io/gitpython-developers/GitPython.png?label=ready&title=Ready)](https://waffle.io/gitpython-developers/GitPython) +[![Throughput Graph](https://graphs.waffle.io/gitpython-developers/GitPython/throughput.svg)](https://waffle.io/gitpython-developers/GitPython/metrics/throughput) Now that there seems to be a massive user base, this should be motivation enough to let git-python return to a proper state, which means diff --git a/doc/source/index.rst b/doc/source/index.rst index 1079c5c76..69fb573a4 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -9,7 +9,6 @@ GitPython Documentation :maxdepth: 2 intro - whatsnew tutorial reference roadmap diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 1c1b0d1b4..1766f8ae0 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -15,7 +15,7 @@ Requirements * `Python`_ 2.7 or newer Since GitPython 2.0.0. Please note that python 2.6 is still reasonably well supported, but might - deteriorate over time. + deteriorate over time. Support is provided on a best-effort basis only. * `Git`_ 1.7.0 or newer It should also work with older versions, but it may be that some operations involving remotes will not work as expected. @@ -75,6 +75,12 @@ codebase for `__del__` implementations and call these yourself when you see fit. Another way assure proper cleanup of resources is to factor out GitPython into a separate process which can be dropped periodically. +Best-effort for Python 2.6 and Windows support +---------------------------------------------- + +This means that support for these platforms is likely to worsen over time +as they are kept alive solely by their users, or not. + Getting Started =============== diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst deleted file mode 100644 index e0d39b099..000000000 --- a/doc/source/whatsnew.rst +++ /dev/null @@ -1,25 +0,0 @@ - -################ -Whats New in 0.3 -################ -GitPython 0.3 is the first step in creating a hybrid which uses a pure python implementations for all simple git features which can be implemented without significant performance penalties. Everything else is still performed using the git command, which is nicely integrated and easy to use. - -Its biggest strength, being the support for all git features through the git command itself, is a weakness as well considering the possibly vast amount of times the git command is being started up. Depending on the actual command being performed, the git repository will be initialized on many of these invocations, causing additional overhead for possibly tiny operations. - -Keeping as many major operations in the python world will result in improved caching benefits as certain data structures just have to be initialized once and can be reused multiple times. This mode of operation may improve performance when altering the git database on a low level, and is clearly beneficial on operating systems where command invocations are very slow. - -**************** -Object Databases -**************** -An object database provides a simple interface to query object information or to write new object data. Objects are generally identified by their 20 byte binary sha1 value during query. - -GitPython uses the ``gitdb`` project to provide a pure-python implementation of the git database, which includes reading and writing loose objects, reading pack files and handling alternate repositories. - -The great thing about this is that ``Repo`` objects can use any object database, hence it easily supports different implementations with different performance characteristics. If you are thinking in extremes, you can implement your own database representation, which may be more efficient for what you want to do specifically, like handling big files more efficiently. - -************************ -Reduced Memory Footprint -************************ -Objects, such as commits, tags, trees and blobs now use 20 byte sha1 signatures internally, reducing their memory demands by 20 bytes per object, allowing you to keep more objects in memory at the same time. - -The internal caches of tree objects were improved to use less memory as well. diff --git a/git/cmd.py b/git/cmd.py index ceea24425..1cc656bf5 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -5,7 +5,6 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php import os -import os.path import sys import select import logging @@ -213,11 +212,11 @@ def _deplete_buffer(fno, handler, buf_list, wg=None): def dashify(string): return string.replace('_', '-') - + def slots_to_dict(self, exclude=()): return dict((s, getattr(self, s)) for s in self.__slots__ if s not in exclude) - + def dict_to_slots_and__excluded_are_none(self, d, excluded=()): for k, v in d.items(): @@ -246,15 +245,15 @@ class Git(LazyMixin): """ __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version_info", "_git_options", "_environment") - + _excluded_ = ('cat_file_all', 'cat_file_header', '_version_info') - + def __getstate__(self): return slots_to_dict(self, exclude=self._excluded_) - + def __setstate__(self, d): dict_to_slots_and__excluded_are_none(self, d, excluded=self._excluded_) - + # CONFIGURATION # The size in bytes read from stdout when copying git's output to another stream max_chunk_size = 1024 * 64 @@ -267,7 +266,7 @@ def __setstate__(self, d): # value of Windows process creation flag taken from MSDN CREATE_NO_WINDOW = 0x08000000 - + # Provide the full path to the git executable. Otherwise it assumes git is in the path _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" GIT_PYTHON_GIT_EXECUTABLE = os.environ.get(_git_exec_env_var, git_exec_name) @@ -339,7 +338,7 @@ def wait(self, stderr=b''): if stderr is None: stderr = b'' stderr = force_bytes(stderr) - + status = self.proc.wait() def read_all_from_possibly_closed_stream(stream): diff --git a/git/index/base.py b/git/index/base.py index 524b4568d..86eda41e6 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -118,13 +118,17 @@ def _set_cache_(self, attr): # read the current index # try memory map for speed lfd = LockedFD(self._file_path) + ok = False try: fd = lfd.open(write=False, stream=False) + ok = True except OSError: - lfd.rollback() # in new repositories, there may be no index, which means we are empty self.entries = dict() return + finally: + if not ok: + lfd.rollback() # END exception handling # Here it comes: on windows in python 2.5, memory maps aren't closed properly @@ -209,8 +213,14 @@ def write(self, file_path=None, ignore_extension_data=False): self.entries lfd = LockedFD(file_path or self._file_path) stream = lfd.open(write=True, stream=True) + ok = False - self._serialize(stream, ignore_extension_data) + try: + self._serialize(stream, ignore_extension_data) + ok = True + finally: + if not ok: + lfd.rollback() lfd.commit() diff --git a/git/test/lib/asserts.py b/git/test/lib/asserts.py index 60a888b3b..9edc49e08 100644 --- a/git/test/lib/asserts.py +++ b/git/test/lib/asserts.py @@ -16,7 +16,10 @@ assert_false ) -from mock import patch +try: + from unittest.mock import patch +except ImportError: + from mock import patch __all__ = ['assert_instance_of', 'assert_not_instance_of', 'assert_none', 'assert_not_none', diff --git a/git/test/lib/helper.py b/git/test/lib/helper.py index 8be2881c3..9488005f4 100644 --- a/git/test/lib/helper.py +++ b/git/test/lib/helper.py @@ -299,6 +299,8 @@ def setUpClass(cls): Dynamically add a read-only repository to our actual type. This way each test type has its own repository """ + import gc + gc.collect() cls.rorepo = Repo(GIT_REPO) @classmethod diff --git a/git/test/performance/test_commit.py b/git/test/performance/test_commit.py index b59c747ee..c60dc2fc4 100644 --- a/git/test/performance/test_commit.py +++ b/git/test/performance/test_commit.py @@ -17,6 +17,10 @@ class TestPerformance(TestBigRepoRW): + def tearDown(self): + import gc + gc.collect() + # ref with about 100 commits in its history ref_100 = '0.1.6' diff --git a/git/test/test_commit.py b/git/test/test_commit.py index c05995033..805221ac1 100644 --- a/git/test/test_commit.py +++ b/git/test/test_commit.py @@ -34,7 +34,11 @@ import os from datetime import datetime from git.objects.util import tzoffset, utc -from mock import Mock + +try: + from unittest.mock import Mock +except ImportError: + from mock import Mock def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False): @@ -343,9 +347,9 @@ def test_gpgsig(self): cstream = BytesIO() cmt._serialize(cstream) assert re.search(r"^gpgsig $", cstream.getvalue().decode('ascii'), re.MULTILINE) - + self.assert_gpgsig_deserialization(cstream) - + cstream.seek(0) cmt.gpgsig = None cmt._deserialize(cstream) @@ -355,27 +359,27 @@ def test_gpgsig(self): cstream = BytesIO() cmt._serialize(cstream) assert not re.search(r"^gpgsig ", cstream.getvalue().decode('ascii'), re.MULTILINE) - + def assert_gpgsig_deserialization(self, cstream): assert 'gpgsig' in 'precondition: need gpgsig' - + class RepoMock: def __init__(self, bytestr): self.bytestr = bytestr - + @property def odb(self): class ODBMock: def __init__(self, bytestr): self.bytestr = bytestr - + def stream(self, *args): stream = Mock(spec_set=['read'], return_value=self.bytestr) stream.read.return_value = self.bytestr return ('binsha', 'typename', 'size', stream) - + return ODBMock(self.bytestr) - + repo_mock = RepoMock(cstream.getvalue()) for field in Commit.__slots__: c = Commit(repo_mock, b'x' * 20) diff --git a/git/test/test_docs.py b/git/test/test_docs.py index b297363dc..2cd355b28 100644 --- a/git/test/test_docs.py +++ b/git/test/test_docs.py @@ -11,6 +11,11 @@ class Tutorials(TestBase): + + def tearDown(self): + import gc + gc.collect() + @with_rw_directory def test_init_repo_object(self, rw_dir): # [1-test_init_repo_object] @@ -64,7 +69,7 @@ def test_init_repo_object(self, rw_dir): assert repo.head.ref == repo.heads.master # head is a symbolic reference pointing to master assert repo.tags['0.3.5'] == repo.tag('refs/tags/0.3.5') # you can access tags in various ways too assert repo.refs.master == repo.heads['master'] # .refs provides access to all refs, i.e. heads ... - + if 'TRAVIS' not in os.environ: assert repo.refs['origin/master'] == repo.remotes.origin.refs.master # ... remotes ... assert repo.refs['0.3.5'] == repo.tags['0.3.5'] # ... and tags diff --git a/git/test/test_git.py b/git/test/test_git.py index b46ac72d6..534539d78 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -6,7 +6,6 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php import os import sys -import mock import subprocess from git.test.lib import ( @@ -28,6 +27,11 @@ from git.compat import PY3 +try: + from unittest import mock +except ImportError: + import mock + class TestGit(TestBase): @@ -36,6 +40,10 @@ def setUpClass(cls): super(TestGit, cls).setUpClass() cls.git = Git(cls.rorepo.working_dir) + def tearDown(self): + import gc + gc.collect() + @patch.object(Git, 'execute') def test_call_process_calls_execute(self, git): git.return_value = '' diff --git a/git/test/test_index.py b/git/test/test_index.py index ca8778388..178a59d2d 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -135,6 +135,25 @@ def _cmp_tree_index(self, tree, index): raise AssertionError("CMP Failed: Missing entries in index: %s, missing in tree: %s" % (bset - iset, iset - bset)) # END assertion message + + @with_rw_repo('0.1.6') + def test_index_lock_handling(self, rw_repo): + def add_bad_blob(): + rw_repo.index.add([Blob(rw_repo, b'f' * 20, 'bad-permissions', 'foo')]) + + try: + ## 1st fail on purpose adding into index. + add_bad_blob() + except Exception as ex: + msg_py3 = "required argument is not an integer" + msg_py2 = "cannot convert argument to integer" + assert msg_py2 in str(ex) or msg_py3 in str(ex) + + ## 2nd time should not fail due to stray lock file + try: + add_bad_blob() + except Exception as ex: + assert "index.lock' could not be obtained" not in str(ex) @with_rw_repo('0.1.6') def test_index_file_from_tree(self, rw_repo): diff --git a/git/test/test_repo.py b/git/test/test_repo.py index e24062c1b..d04a0f66f 100644 --- a/git/test/test_repo.py +++ b/git/test/test_repo.py @@ -115,6 +115,7 @@ def test_commit_from_revision(self): assert commit.type == 'commit' assert self.rorepo.commit(commit) == commit + def test_commits(self): mc = 10 commits = list(self.rorepo.iter_commits('0.1.6', max_count=mc)) assert len(commits) == mc diff --git a/setup.py b/setup.py index 05c12b8f2..b3b43eb3b 100755 --- a/setup.py +++ b/setup.py @@ -68,8 +68,10 @@ def _stamp_version(filename): print("WARNING: Couldn't find version line in file %s" % filename, file=sys.stderr) install_requires = ['gitdb >= 0.6.4'] +test_requires = ['node'] if sys.version_info[:2] < (2, 7): install_requires.append('ordereddict') + test_requires.append('mock') # end setup( @@ -87,7 +89,7 @@ def _stamp_version(filename): license="BSD License", requires=['gitdb (>=0.6.4)'], install_requires=install_requires, - test_requirements=['mock', 'nose'] + install_requires, + test_requirements=test_requires + install_requires, zip_safe=False, long_description="""\ GitPython is a python library used to interact with Git repositories""",