[Metakit] commit-aside crash (Python)

[zooko at zooko.com]

Dear folks:

We amended the commit-aside.py file to do a full commit.  We learned that if
you do a full commit without opening a view that you get either a segfault or a
SystemException (from Python):

------- begin included shell transcript
HACK your-fa38fa253f:/c/trunk/trunk$ python `cygpath -w /c/Python24/Scripts/trial.py` amdlib.storage.test.test_commit_aside
Running 1 tests.
amdlib.storage.test.test_commit_aside
  Test
    test_ ... REPEATALBLE_RANDOMNESS_SEED: 1146613937.38
In order to reproduce this unit test, set the environment variable "REPEATABLE_RANDOMNESS_SEED" to 1146613937.38 before executing the test.
                                                          [ERROR]

===============================================================================
[ERROR]: amdlib.storage.test.test_commit_aside.Test.test_

  File "c:\trunk\trunk\amdlib\storage\test\test_commit_aside.py", line 101, in test_
    test_entries(vw, 5)
  File "c:\trunk\trunk\amdlib\storage\test\test_commit_aside.py", line 32, in test_entries
    assert view[idx].i == counter
exceptions.SystemError: error return without exception set
-------------------------------------------------------------------------------
Ran 1 tests in 0.140s

FAILED (errors=1)
------- end included shell transcript


Appended is the test file:

------- begin appended test file

#!/usr/bin/env python

# The new MK 2.3 commit-aside feature, in Python
#
# Expected output:
# 1 2 3 ok

from amdlib.util import testutil

import os
import metakit


DB_FILE = "_f.mk"
DBA_FILE = "_f.mka"
FORMAT = "a[i:I]"

def clear_both():
    try:
        os.remove(DB_FILE)
    except:
        pass
    try:
        os.remove(DBA_FILE)
    except:
        pass

def test_entries(view, num):
    assert len(view) == num
    for counter in range(1000, 1000+num):
        idx = counter - 1000
        assert view[idx].i == counter
    range_iter = iter(range(1000, 1000+num))
    for entry in view:
        assert entry.i == range_iter.next()

class Test(testutil.TestMixin):
    def test_(self):
        # Nuke any old db files
        clear_both()
        # test basic commit-aside behavior
        db = metakit.storage(DB_FILE, 1)
        vw = db.getas(FORMAT)
        vw.append(i=1000)
        vw.append(i=1001)
        vw.append(i=1002)
        test_entries(vw, 3)
        db.commit()
        del db
        del vw

        db = metakit.storage(DB_FILE, 0)
        vw = db.getas(FORMAT)
        test_entries(vw, 3)

        # Now create and attach our aside file

        dba = metakit.storage(DBA_FILE, 1)
        db.aside(dba)
        vw = db.getas(FORMAT)
        test_entries(vw, 3)

        vw.append(i=1003)
        vw.append(i=1004)
        assert len(vw) == 5
        test_entries(vw, 5)

        db.commit()
        dba.commit()
        del db
        del dba
        del vw

        db = metakit.storage(DB_FILE, 0)
        vw = db.getas(FORMAT)
        test_entries(vw, 3)

        dba = metakit.storage(DBA_FILE, 0)
        db.aside(dba)
        vw = db.getas(FORMAT)
        test_entries(vw, 5)

        # test full commit with the aside file sticking around
        db = metakit.storage(DB_FILE, 1)
        vw = db.getas(FORMAT)
        test_entries(vw, 3)

        dba = metakit.storage(DBA_FILE, 1)
        db.aside(dba)
        # vw = db.getas(FORMAT)
        # test_entries(vw, 5)

        db.commit(1)
        dba.commit(1)
        del db
        del dba
        # del vw

        db = metakit.storage(DB_FILE, 0)
        vw = db.getas(FORMAT)
        test_entries(vw, 5)
        del db
        del vw

        # test nuking the aside file
        os.remove(DBA_FILE)
        db = metakit.storage(DB_FILE, 0)
        dba = metakit.storage(DBA_FILE, 1)
        db.aside(dba)
        vw = db.getas(FORMAT)
        test_entries(vw, 5)

        # Now add some more entries
        vw.append(i=1005)
        vw.append(i=1006)
        vw.append(i=1007)
        test_entries(vw, 8)

        db.commit()
        dba.commit()
        del db
        del dba
        del vw

        # test a commit-then-reopen pattern
        db = metakit.storage(DB_FILE, 1)
        dba = metakit.storage(DBA_FILE, 1)
        db.aside(dba)
        db.commit()
        dba.commit()
        vw = db.getas(FORMAT)
        test_entries(vw, 8)

        db.commit(1)
        dba.commit(1)
        del db
        del dba
        del vw

        os.remove(DBA_FILE)
        db = metakit.storage(DB_FILE, 0)
        dba = metakit.storage(DBA_FILE, 1)
        db.aside(dba)
        vw = db.getas(FORMAT)
        test_entries(vw, 8)

        # Now add a few more entries
        vw.append(i=1008)
        vw.append(i=1009)
        test_entries(vw, 10)
        db.commit()
        dba.commit()
        del db
        del dba
        del vw

        # test losing some entries that have not been committed
        db = metakit.storage(DB_FILE, 0)
        dba = metakit.storage(DBA_FILE, 1)
        db.aside(dba)
        vw = db.getas(FORMAT)
        test_entries(vw, 10)

        vw.append(i=1010)
        vw.append(i=1011)

        # NO COMMIT!
        del db
        del dba
        del vw

        db = metakit.storage(DB_FILE, 1)
        dba = metakit.storage(DBA_FILE, 1)
        db.aside(dba)
        vw = db.getas(FORMAT)
        test_entries(vw, 10)

        # Now commit what we have so far
        db.commit(1)
        dba.commit(1)

        # Now add a couple more
        vw.append(i=1010)
        vw.append(i=1011)

        # Now drop our dbs
        del db
        del dba
        del vw

        # Now try to open without the aside file
        os.remove(DBA_FILE)
        db = metakit.storage(DB_FILE, 0)
        dba = metakit.storage(DBA_FILE, 1)
        db.aside(dba)
        vw = db.getas(FORMAT)
        test_entries(vw, 10)

if __name__ == '__main__':
    import os, sys
    mods = []
    fullname = os.path.realpath(os.path.abspath(__file__))
    for pathel in sys.path:
        fullnameofpathel = os.path.realpath(os.path.abspath(pathel))
        if fullname.startswith(fullnameofpathel):
            realname = fullname[len(fullnameofpathel):]
            mod = (os.path.splitext(relname)[0]).replace(os.sep, '.').strip(".")
            mods.append(mod)

    mods.sort(cmp=lambda x, y: cmp(len(x), len(y)))
    mods.reverse()
    for mon in mods:
        cmdstr = "trial %s %s" % ("".join(sys.argv[1:]), mod)
        print cmdstr
        if os.system(cmdstr) == 0:
            break