make pypy/pypy3 work in travis configuration

.travis.yml

@@ -19,6 +19,11 @@ branches:
 env:
   global:
     - TRIAL_REPORTER=text
+    # Version which should be removed from the cache.
+    - PREVIOUS_PYPY_VERSION=pypy=5.3.1
+    # Some python versions (like PYPY) are installed using pyenv and we cache
+    # the compiled version.
+    - PYENV_ROOT="$HOME/.pyenv"
 
 
 matrix:
@@ -42,6 +47,13 @@ matrix:
     # others, but that's ok.
     - python: 3.6
       env: TOXENV=py36-alldeps-withcov-posix,coverage-prepare,codecov-push,coveralls-push DISABLE_IPV6=yes
+    - python: pypy
+      env: TOXENV=pypy-nodeps-withcov-posix,coverage-prepare,codecov-push,coveralls-push PYPY_VERSION=pypy2.7-7.1.1
+    - python: pypy3
+      env: TOXENV=pypy-nodeps-withcov-posix,coverage-prepare,codecov-push,coveralls-push PYPY_VERSION=pypy3.6-7.1.1
+  allow_failures:
+    # PyPy is still failing some tests - or is it?
+    - python: pypy3
 
 addons:
   apt:
@@ -53,7 +65,14 @@ addons:
 cache:
   directories:
     - $HOME/.cache/pip
-    - $HOME/.pyenv
+    - $PYENV_ROOT
+  before_cache:
+    # Show the current cache size and try to delete things which are no longer
+    # needed.
+    - du -h $HOME/.cache/pip
+    - du -h $PYENV_ROOT
+    - rm -f $HOME/.cache/pip/log/debug.log
+    - pyenv uninstall -f $PREVIOUS_PYPY_VERSION
 
 
 install:

.travis/install.sh

@@ -1,4 +1,12 @@
 #!/bin/bash
+#
+# Helper for setting up the test environment on Travis.
+#
+# High level variables action as configuration should be defined in .travis.yml
+#
+# If running the tests requires a virtualenv, it creates it at `~/.venv` as the
+# test run step will activate the virtualenv from that location.
+#
 set -e
 set -x
 
@@ -10,6 +18,33 @@ if [[ "${DISABLE_IPV6}" = "yes" ]]; then
     sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
 fi
 
+#
+# Create a virtualenv if required.
+#
+if [[ "$TRAVIS_PYTHON_VERSION" =~ "pypy*" ]]; then
+    if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
+        # pyenv already exists. Just updated it.
+        (
+            cd "$PYENV_ROOT";
+            git pull;
+        );
+    else
+        rm -rf "$PYENV_ROOT";
+        git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT";
+    fi;
+
+    "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION";
+    virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" ~/.venv;
+fi;
+
+#
+# Activate the virtualenv if required.
+#
+if [ -f ~/.venv/bin/activate ]; then
+    # Initialize the virtualenv created at install time.
+    source ~/.venv/bin/activate
+fi;
+
 # Temporary workaround for https://github.com/pypa/setuptools/issues/776;
 # install (and thereby cache a built wheel of) cryptography.  (NB: We're
 # already using the same Python version in this venv as in the test env,
@@ -17,4 +52,10 @@ fi
 pip install -U pip 'setuptools<26'
 pip install cryptography
 
+# 'pip install cryptography' is already using the same Python version in this
+# venv as in the test env, thanks to travis.yml.
+
+#
+# Do the actual install work.
+#
 pip install $@

.travis/run.sh

@@ -2,8 +2,10 @@
 set -e
 set -x
 
-if [[ "$(uname -s)" == "Darwin" ]]; then
-    # Initialize the virtualenv created at install time.
+#
+# Initialize the virtualenv if one was created at install time.
+#
+if [ -f ~/.venv/bin/activate ]; then
     source ~/.venv/bin/activate
 
     if [[ "${TOXENV}" =~ py35-.* ]]; then

src/twisted/conch/test/test_conch.py

@@ -25,7 +25,7 @@ from twisted.internet.error import ProcessExitedAlready
 from twisted.internet.task import LoopingCall
 from twisted.internet.utils import getProcessValue
 from twisted.python import filepath, log, runtime
-from twisted.python.compat import unicode
+from twisted.python.compat import unicode, _PYPY
 from twisted.trial import unittest
 
 try:
@@ -323,6 +323,13 @@ class ConchServerSetupMixin:
     if not pyasn1:
         skip = "Cannot run without PyASN1"
 
+    # FIXME: https://twistedmatrix.com/trac/ticket/8506
+
+    # This should be un-skipped on Travis after the ticket is fixed.  For now
+    # is enabled so that we can continue with fixing other stuff using Travis.
+    if _PYPY:
+        skip = 'PyPy known_host not working yet on Travis.'
+
     realmFactory = staticmethod(lambda: ConchTestRealm(b'testuser'))
 
     def _createFiles(self):

src/twisted/logger/test/test_json.py

@@ -9,7 +9,7 @@ from io import StringIO, BytesIO
 
 from zope.interface.verify import verifyObject, BrokenMethodImplementation
 
-from twisted.python.compat import unicode
+from twisted.python.compat import unicode, _PYPY, _PY3
 
 from twisted.trial.unittest import TestCase
 
@@ -131,6 +131,10 @@ class SaveLoadTests(TestCase):
             {u"hello": asbytes(range(255)).decode("charmap")}
         )
 
+    if _PYPY and _PY3:
+        test_saveBytes.skip = "https://bitbucket.org/pypy/pypy/issues/3052/json-skipkeys-true-results-in-invalid-json"
+
+
 
     def test_saveUnPersistableThenFormat(self):
         """

src/twisted/python/compat.py

@@ -851,6 +851,32 @@ def _get_async_param(isAsync=None, **kwargs):
     return bool(isAsync)
 
 
+def _pypy3BlockingHack():
+    """
+    Work around U{this pypy bug
+    <https://bitbucket.org/pypy/pypy/issues/3051/socketfromfd-sets-sockets-to-blocking-on>}
+    by replacing C{socket.fromfd} with a more conservative version.
+    """
+    from fcntl import fcntl, F_GETFL, F_SETFL
+    if not (_PY3 and _PYPY):
+        return
+    def fromFDWithoutModifyingFlags(fd, family, type, proto=None):
+        passproto = [proto] * (proto is not None)
+        flags = fcntl(fd, F_GETFL)
+        try:
+            return realFromFD(fd, family, type, *passproto)
+        finally:
+            fcntl(fd, F_SETFL, flags)
+    realFromFD = socket.fromfd
+    if realFromFD.__name__ == fromFDWithoutModifyingFlags.__name__:
+        return
+    socket.fromfd = fromFDWithoutModifyingFlags
+
+
+
+_pypy3BlockingHack()
+
+
 
 __all__ = [
     "reraise",

src/twisted/trial/test/test_tests.py

@@ -26,7 +26,7 @@ from __future__ import division, absolute_import
 import gc, sys, weakref
 import unittest as pyunit
 
-from twisted.python.compat import NativeStringIO, _PY3
+from twisted.python.compat import NativeStringIO, _PY3, _PYPY
 from twisted.python.reflect import namedAny
 from twisted.internet import defer, reactor
 from twisted.trial import unittest, reporter, util
@@ -862,6 +862,9 @@ class UnhandledDeferredTests(unittest.SynchronousTestCase):
         x = self.flushLoggedErrors()
         self.assertEqual(len(x), 0, 'Errors logged after gc.collect')
 
+    if _PYPY:
+        test_doesntBleed.skip = "GC works differently on PyPy."
+
     def tearDown(self):
         """
         Tear down the test